mtmd : drop `_shared` from `libmtmd` name, merge helpers into libmtmd (⚠️ breaking change) #13917

ngxson · 2025-05-30T10:13:25Z

Supersede #13903

Small breaking changes:

libmtmd_shared.dll/so/dyblib is now renamed to libmtmd.dll/so/dyblib, no more _shared postfix
merge helpers function back to libmtmd, revert the changes in mtmd : move helpers to dedicated library (⚠️ breaking change) #13866

@65a This is made to closely resemble how libllama is compiled:

no _shared prefix
no OBJECT when add_library
no STATIC declaration

So you should be able to link it exactly the same way as libllama. But please lmk if there are other problems.

@ggerganov I'm not very confident about cmake stuff, could you please take a look? Thanks.

JamePeng · 2025-05-30T10:39:02Z

It is recommended to merge them into a dynamic library such as libmtmd.dll to avoid the last submission where mtmd and mtmd_helper were separated, resulting in the library function not being able to find the MTMD_API function in mtmd_helper.

ggerganov · 2025-05-30T11:30:38Z

Here is how I think it should build:

diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt
index edea27a63..382d82033 100644
--- a/tools/mtmd/CMakeLists.txt
+++ b/tools/mtmd/CMakeLists.txt
@@ -1,5 +1,7 @@
 # mtmd
 
+find_package(Threads REQUIRED)
+
 add_library(mtmd
             mtmd.cpp
             mtmd-audio.cpp
@@ -9,46 +11,50 @@ add_library(mtmd
             clip-impl.h
             )
 
-target_link_libraries(mtmd PRIVATE ggml llama ${CMAKE_THREAD_LIBS_INIT})
-target_include_directories(mtmd PUBLIC .)
+target_link_libraries     (mtmd PUBLIC ggml llama)
+target_link_libraries     (mtmd PRIVATE Threads::Threads)
+target_include_directories(mtmd PUBLIC  .)
 target_include_directories(mtmd PRIVATE ../..)
-target_compile_features(mtmd PRIVATE cxx_std_17)
+target_compile_features   (mtmd PRIVATE cxx_std_17)
+
+if (BUILD_SHARED_LIBS)
+    set_target_properties     (mtmd PROPERTIES POSITION_INDEPENDENT_CODE ON)
+    target_compile_definitions(mtmd PRIVATE LLAMA_BUILD)
+    target_compile_definitions(mtmd PUBLIC  LLAMA_SHARED)
+endif()
+
+set(MTMD_PUBLIC_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/mtmd.h)
+
+set_target_properties(mtmd
+    PROPERTIES
+    PUBLIC_HEADER "${MTMD_PUBLIC_HEADERS}")
+
+install(TARGETS mtmd LIBRARY PUBLIC_HEADER)
 
 # compile the helper separately, to avoid long compile times with miniaudio.h and stb_image.h
 
-add_library(mtmd_helper
+add_library(mtmd-helper STATIC
             mtmd-helper.cpp
             mtmd-helper.h
             )
 
-target_link_libraries(mtmd_helper PRIVATE ggml llama mtmd ${CMAKE_THREAD_LIBS_INIT})
-target_include_directories(mtmd_helper PUBLIC .)
-target_include_directories(mtmd_helper PRIVATE ./vendor)
-target_include_directories(mtmd_helper PRIVATE ../..)
-target_compile_features(mtmd_helper PRIVATE cxx_std_17)
+target_link_libraries     (mtmd-helper PRIVATE mtmd Threads::Threads)
+target_include_directories(mtmd-helper PUBLIC  .)
+target_include_directories(mtmd-helper PRIVATE ../..)
+target_compile_features   (mtmd-helper PRIVATE cxx_std_17)
 
 if (BUILD_SHARED_LIBS)
-    set_target_properties(mtmd PROPERTIES POSITION_INDEPENDENT_CODE ON)
-    target_compile_definitions(mtmd PRIVATE LLAMA_SHARED LLAMA_BUILD)
-    target_link_libraries(mtmd PRIVATE ggml llama ${CMAKE_THREAD_LIBS_INIT})
-    set_target_properties(mtmd PROPERTIES PUBLIC_HEADER "mtmd.h")
-    install(TARGETS mtmd LIBRARY PUBLIC_HEADER)
-
-    set_target_properties(mtmd_helper PROPERTIES POSITION_INDEPENDENT_CODE ON)
-    target_compile_definitions(mtmd_helper PRIVATE LLAMA_SHARED LLAMA_BUILD)
-    target_link_libraries(mtmd_helper PRIVATE ggml llama mtmd ${CMAKE_THREAD_LIBS_INIT})
-    set_target_properties(mtmd_helper PROPERTIES PUBLIC_HEADER "mtmd-helper.h")
-    install(TARGETS mtmd_helper LIBRARY PUBLIC_HEADER)
+    set_target_properties     (mtmd-helper PROPERTIES POSITION_INDEPENDENT_CODE ON)
 endif()
 
 if (NOT MSVC)
     # for stb_image.h and miniaudio.h
-    target_compile_options(mtmd_helper PRIVATE -Wno-cast-qual)
+    target_compile_options(mtmd-helper PRIVATE -Wno-cast-qual)
 endif()
 
-if(TARGET BUILD_INFO)
-    add_dependencies(mtmd BUILD_INFO)
-    add_dependencies(mtmd_helper BUILD_INFO)
+if (TARGET BUILD_INFO)
+    add_dependencies(mtmd        BUILD_INFO)
+    add_dependencies(mtmd-helper BUILD_INFO)
 endif()
 
 add_executable(llama-llava-cli    deprecation-warning.cpp)
@@ -57,8 +63,8 @@ add_executable(llama-minicpmv-cli deprecation-warning.cpp)
 add_executable(llama-qwen2vl-cli  deprecation-warning.cpp)
 
 set(TARGET llama-mtmd-cli)
-add_executable(${TARGET} mtmd-cli.cpp)
-set_target_properties(${TARGET} PROPERTIES OUTPUT_NAME llama-mtmd-cli)
-install(TARGETS ${TARGET} RUNTIME)
-target_link_libraries(${TARGET} PRIVATE common mtmd mtmd_helper ${CMAKE_THREAD_LIBS_INIT})
+add_executable         (${TARGET} mtmd-cli.cpp)
+set_target_properties  (${TARGET} PROPERTIES OUTPUT_NAME llama-mtmd-cli)
+install                (TARGETS ${TARGET} RUNTIME)
+target_link_libraries  (${TARGET} PRIVATE common mtmd mtmd-helper Threads::Threads)
 target_compile_features(${TARGET} PRIVATE cxx_std_17)
diff --git a/tools/server/CMakeLists.txt b/tools/server/CMakeLists.txt
index 08597145c..00fdcaca6 100644
--- a/tools/server/CMakeLists.txt
+++ b/tools/server/CMakeLists.txt
@@ -36,7 +36,7 @@ install(TARGETS ${TARGET} RUNTIME)
 
 target_include_directories(${TARGET} PRIVATE ../llava)
 target_include_directories(${TARGET} PRIVATE ${CMAKE_SOURCE_DIR})
-target_link_libraries(${TARGET} PRIVATE common mtmd mtmd_helper ${CMAKE_THREAD_LIBS_INIT})
+target_link_libraries(${TARGET} PRIVATE common mtmd mtmd-helper ${CMAKE_THREAD_LIBS_INIT})
 
 if (LLAMA_SERVER_SSL)
     find_package(OpenSSL REQUIRED)

You want libmtmd-helper to always be statically linked - it's not supposed to be used by 3rd parties. Also added mtmd.h to be installed as a public header.

ngxson · 2025-05-30T12:51:44Z

You want libmtmd-helper to always be statically linked - it's not supposed to be used by 3rd parties. Also added mtmd.h to be installed as a public header.

The helper is supposed to be used by downstream projects. Without it, it would be quite difficult to convince developers to move to libmtmd. In other words, my idea is that helpers can be part of public API, but not part of the "stable" API.

On second thought, I think it will be less confusing for developers if I ship everything in one single lib, while still separate them internally.

I did that in ed4127c , does it looks good to you @ggerganov ?

Also, I realized that my assumption was not correct:

# compile the helper separately, to avoid long compile times with miniaudio.h and stb_image.h

Indeed, they are compiled as different objects:

[ 71%] Building CXX object tools/mtmd/CMakeFiles/mtmd.dir/mtmd.cpp.o
[ 75%] Building CXX object tools/mtmd/CMakeFiles/mtmd.dir/mtmd-audio.cpp.o
[ 78%] Building CXX object common/CMakeFiles/common.dir/arg.cpp.o
[ 81%] Building CXX object tools/mtmd/CMakeFiles/mtmd.dir/mtmd-helper.cpp.o

So for example, if I change the code in mtmd.cpp, the mtmd-helper.cpp won't be re-compiled. Therefore, I think I can revert my change earlier that separated them into 2 different libs

65a · 2025-05-30T16:06:08Z

Works perfectly from here. Thanks for the fix. Feel free to mark it as fixes 13902, I'll drop my pull.

ngxson added 2 commits May 30, 2025 11:58

mtmd : fix missing public header

153e18d

no object

7598fa4

ngxson requested a review from ggerganov May 30, 2025 10:13

github-actions bot added the examples label May 30, 2025

ngxson added 2 commits May 30, 2025 14:22

apply suggestion from Georgi

39db27c

rm mtmd-helper, merge it to mtmd

ed4127c

github-actions bot added the server label May 30, 2025

ggerganov approved these changes May 30, 2025

View reviewed changes

ngxson changed the title ~~mtmd : cmake of libmtmd now closely resemble libllama (⚠️ breaking change)~~ mtmd : drop _shared prefix from libmtmdname (⚠️ breaking change) May 30, 2025

ngxson changed the title ~~mtmd : drop _shared prefix from libmtmdname (⚠️ breaking change)~~ mtmd : drop _shared from libmtmdname, merge helpers into libmtmd (⚠️ breaking change) May 30, 2025

ngxson changed the title ~~mtmd : drop _shared from libmtmdname, merge helpers into libmtmd (⚠️ breaking change)~~ mtmd : drop _shared from libmtmd name, merge helpers into libmtmd (⚠️ breaking change) May 30, 2025

ngxson added 2 commits May 30, 2025 16:59

Merge branch 'master' into xsn/mtmd_fix_pub_header

0f479b8

missing vendor include dir

1baa12f

65a mentioned this pull request May 30, 2025

cmake: mtmd: install PUBLIC_HEADERs and allow static linking (#13902) #13903

Closed

ngxson merged commit 51fa76f into ggml-org:master May 31, 2025
46 checks passed

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

mtmd : drop `_shared` from `libmtmd` name, merge helpers into libmtmd (⚠️ breaking change) #13917

mtmd : drop `_shared` from `libmtmd` name, merge helpers into libmtmd (⚠️ breaking change) #13917

Uh oh!

ngxson commented May 30, 2025 •

edited

Loading

Uh oh!

JamePeng commented May 30, 2025

Uh oh!

ggerganov commented May 30, 2025

Uh oh!

ngxson commented May 30, 2025 •

edited

Loading

Uh oh!

65a commented May 30, 2025

Uh oh!

Uh oh!

Uh oh!

mtmd : drop _shared from libmtmd name, merge helpers into libmtmd (⚠️ breaking change) #13917

mtmd : drop _shared from libmtmd name, merge helpers into libmtmd (⚠️ breaking change) #13917

Uh oh!

Conversation

ngxson commented May 30, 2025 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

JamePeng commented May 30, 2025

Uh oh!

ggerganov commented May 30, 2025

Uh oh!

ngxson commented May 30, 2025 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

65a commented May 30, 2025

Uh oh!

Uh oh!

Uh oh!

mtmd : drop `_shared` from `libmtmd` name, merge helpers into libmtmd (⚠️ breaking change) #13917

mtmd : drop `_shared` from `libmtmd` name, merge helpers into libmtmd (⚠️ breaking change) #13917

ngxson commented May 30, 2025 •

edited

Loading

ngxson commented May 30, 2025 •

edited

Loading