whisper.objc : enable Core ML in example & fix segmentation fault (#910)

* coreml : update endcoder header import path

* coreml : force objc_arc in whisper-encoder.mm

* whisper.objc : create coreml/ group link

* whisper.objc : add coreml model link

* whisper.objc : update readme

* coreml : use -fobjc-arc for coreml/whisper-encoder.mm

* ci: create dummy .mlmodelc for pass ios build

* whisper.objc : update readme

---------

Co-authored-by: Georgi Gerganov <redacted>

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 02c8900d420c3100fceaa64bf2a64829e3d4b8d8..657570077302bfdd0a5909b57f83c567d7a9c861 100644 (file)
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -333,7 +333,9 @@ jobs:
         uses: actions/checkout@v1
 
       - name: Configure
-        run: cp models/for-tests-ggml-base.en.bin models/ggml-base.en.bin
+        run: |
+          cp models/for-tests-ggml-base.en.bin models/ggml-base.en.bin
+          mkdir models/ggml-base.en-encoder.mlmodelc
 
       - name: Build objc example
         run: xcodebuild -project examples/whisper.objc/whisper.objc.xcodeproj -scheme whisper.objc -configuration ${{ matrix.build }} -sdk iphonesimulator build

diff --git a/Makefile b/Makefile
index 078713620df6ba01d8d5f472e50a5783bb38ee51..7bb7e31ae0f78f80ef52b83b3f15973ff9c292e1 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -240,7 +240,7 @@ ifndef WHISPER_COREML
 WHISPER_OBJ += whisper.o
 else
 whisper-encoder.o: coreml/whisper-encoder.mm coreml/whisper-encoder.h
-       $(CXX) -O3 -I . -c coreml/whisper-encoder.mm -o whisper-encoder.o
+       $(CXX) -O3 -I . -fobjc-arc -c coreml/whisper-encoder.mm -o whisper-encoder.o
 
 whisper-encoder-impl.o: coreml/whisper-encoder-impl.m coreml/whisper-encoder-impl.h
        $(CXX) -O3 -I . -fobjc-arc -c coreml/whisper-encoder-impl.m -o whisper-encoder-impl.o

diff --git a/coreml/whisper-encoder.mm b/coreml/whisper-encoder.mm
index 2fcccea1875c873af7c212a7afb8cf8c704f5389..09e44fcbf890fc0fd4828bb04fcbc25ce4abb84a 100644 (file)
--- a/coreml/whisper-encoder.mm
+++ b/coreml/whisper-encoder.mm
@@ -1,5 +1,9 @@
-#import "coreml/whisper-encoder.h"
-#import "coreml/whisper-encoder-impl.h"
+#if !__has_feature(objc_arc)
+#error This file must be compiled with automatic reference counting enabled (-fobjc-arc)
+#endif
+
+#import "whisper-encoder.h"
+#import "whisper-encoder-impl.h"
 
 #import <CoreML/CoreML.h>
 
@@ -52,8 +56,6 @@ void whisper_coreml_encode(
     whisper_encoder_implOutput * outCoreML = [(__bridge id) ctx->data predictionFromLogmel_data:inMultiArray error:nil];
 
     memcpy(out, outCoreML.output.dataPointer, outCoreML.output.count * sizeof(float));
-
-    [inMultiArray release];
 }
 
 #if __cplusplus

diff --git a/examples/whisper.objc/README.md b/examples/whisper.objc/README.md
index 46174d2de85213b7eb83dc97a9436401215a2bcd..6833ebb7e3b5eeba4aa249716ed0201c70de9773 100644 (file)
--- a/examples/whisper.objc/README.md
+++ b/examples/whisper.objc/README.md
@@ -14,15 +14,24 @@ https://user-images.githubusercontent.com/1991296/204126266-ce4177c6-6eca-4bd9-b
 ```java
 git clone https://github.com/ggerganov/whisper.cpp
 open whisper.cpp/examples/whisper.objc/whisper.objc.xcodeproj/
+
+// If you don't want to convert a Core ML model, you can skip this step by create dummy model
+mkdir models/ggml-base.en-encoder.mlmodelc
 ```
 
 Make sure to build the project in `Release`:
 
 <img width="947" alt="image" src="https://user-images.githubusercontent.com/1991296/197382607-9e1e6d1b-79fa-496f-9d16-b71dc1535701.png">
 
-Also, don't forget to add the `-DGGML_USE_ACCELERATE` compiler flag in Build Phases.
+Also, don't forget to add the `-DGGML_USE_ACCELERATE` compiler flag for `ggml.c` in Build Phases.
 This can significantly improve the performance of the transcription:
 
 <img width="1072" alt="image" src="https://user-images.githubusercontent.com/1991296/208511239-8d7cdbd1-aa48-41b5-becd-ca288d53cc07.png">
 
+If you want to enable Core ML support, you can add the `-DWHISPER_USE_COREML -DWHISPER_COREML_ALLOW_FALLBACK` compiler flag for `whisper.cpp` in Build Phases:
+
+<img width="1072" alt="image" src="https://github.com/ggerganov/whisper.cpp/assets/3001525/103e8f57-6eb6-490d-a60c-f6cf6c319324">
+
+Then follow the [`Core ML support` section of readme](../../README.md#core-ml-support) for convert the model.
+
 In this project, it also added `-O3 -DNDEBUG` to `Other C Flags`, but adding flags to app proj is not ideal in real world (applies to all C/C++ files), consider splitting xcodeproj in workspace in your own project.

diff --git a/examples/whisper.objc/whisper.objc.xcodeproj/project.pbxproj b/examples/whisper.objc/whisper.objc.xcodeproj/project.pbxproj
index 2c93de759d6cbea70b7becb3766e02c80a688592..49bd74e6326835e216c071bfac683b0f31d5d218 100644 (file)
--- a/examples/whisper.objc/whisper.objc.xcodeproj/project.pbxproj
+++ b/examples/whisper.objc/whisper.objc.xcodeproj/project.pbxproj
@@ -14,9 +14,13 @@
                18627C8629052BE000BD2A04 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 18627C8529052BE000BD2A04 /* Assets.xcassets */; };
                18627C8929052BE000BD2A04 /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 18627C8729052BE000BD2A04 /* LaunchScreen.storyboard */; };
                18627C8C29052BE000BD2A04 /* main.m in Sources */ = {isa = PBXBuildFile; fileRef = 18627C8B29052BE000BD2A04 /* main.m */; };
-               18627C9429052C4900BD2A04 /* whisper.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 18627C9329052C4900BD2A04 /* whisper.cpp */; };
+               18627C9429052C4900BD2A04 /* whisper.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 18627C9329052C4900BD2A04 /* whisper.cpp */; settings = {COMPILER_FLAGS = "-DWHISPER_USE_COREML -DWHISPER_COREML_ALLOW_FALLBACK"; }; };
                18627C9629052C5800BD2A04 /* ggml.c in Sources */ = {isa = PBXBuildFile; fileRef = 18627C9529052C5800BD2A04 /* ggml.c */; settings = {COMPILER_FLAGS = "-DGGML_USE_ACCELERATE"; }; };
                18627C9B29052CFF00BD2A04 /* ggml-base.en.bin in Resources */ = {isa = PBXBuildFile; fileRef = 18627C9A29052CFF00BD2A04 /* ggml-base.en.bin */; };
+               7FE3424B2A0C3FA20015A058 /* whisper-encoder-impl.m in Sources */ = {isa = PBXBuildFile; fileRef = 7FE342452A0C3FA20015A058 /* whisper-encoder-impl.m */; };
+               7FE3424C2A0C3FA20015A058 /* whisper-encoder.mm in Sources */ = {isa = PBXBuildFile; fileRef = 7FE342472A0C3FA20015A058 /* whisper-encoder.mm */; };
+               7FE3424D2A0C3FA20015A058 /* whisper-decoder-impl.m in Sources */ = {isa = PBXBuildFile; fileRef = 7FE3424A2A0C3FA20015A058 /* whisper-decoder-impl.m */; };
+               7FE3424F2A0C418A0015A058 /* ggml-base.en-encoder.mlmodelc in Resources */ = {isa = PBXBuildFile; fileRef = 7FE3424E2A0C418A0015A058 /* ggml-base.en-encoder.mlmodelc */; };
 /* End PBXBuildFile section */
 
 /* Begin PBXFileReference section */
@@ -37,6 +41,13 @@
                18627C9529052C5800BD2A04 /* ggml.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = ggml.c; path = ../../../ggml.c; sourceTree = "<group>"; };
                18627C9729052C6600BD2A04 /* ggml.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = ggml.h; path = ../../../ggml.h; sourceTree = "<group>"; };
                18627C9A29052CFF00BD2A04 /* ggml-base.en.bin */ = {isa = PBXFileReference; lastKnownFileType = archive.macbinary; name = "ggml-base.en.bin"; path = "../../../models/ggml-base.en.bin"; sourceTree = "<group>"; };
+               7FE342452A0C3FA20015A058 /* whisper-encoder-impl.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = "whisper-encoder-impl.m"; sourceTree = "<group>"; };
+               7FE342462A0C3FA20015A058 /* whisper-encoder.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "whisper-encoder.h"; sourceTree = "<group>"; };
+               7FE342472A0C3FA20015A058 /* whisper-encoder.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = "whisper-encoder.mm"; sourceTree = "<group>"; };
+               7FE342482A0C3FA20015A058 /* whisper-decoder-impl.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "whisper-decoder-impl.h"; sourceTree = "<group>"; };
+               7FE342492A0C3FA20015A058 /* whisper-encoder-impl.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "whisper-encoder-impl.h"; sourceTree = "<group>"; };
+               7FE3424A2A0C3FA20015A058 /* whisper-decoder-impl.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = "whisper-decoder-impl.m"; sourceTree = "<group>"; };
+               7FE3424E2A0C418A0015A058 /* ggml-base.en-encoder.mlmodelc */ = {isa = PBXFileReference; lastKnownFileType = wrapper; name = "ggml-base.en-encoder.mlmodelc"; path = "../../../models/ggml-base.en-encoder.mlmodelc"; sourceTree = "<group>"; };
 /* End PBXFileReference section */
 
 /* Begin PBXFrameworksBuildPhase section */
@@ -69,6 +80,8 @@
                18627C7829052BDF00BD2A04 /* whisper.objc */ = {
                        isa = PBXGroup;
                        children = (
+                               7FE3424E2A0C418A0015A058 /* ggml-base.en-encoder.mlmodelc */,
+                               7FE342442A0C3FA20015A058 /* coreml */,
                                18627C9A29052CFF00BD2A04 /* ggml-base.en.bin */,
                                18627C9729052C6600BD2A04 /* ggml.h */,
                                18627C9529052C5800BD2A04 /* ggml.c */,
@@ -89,6 +102,20 @@
                        path = whisper.objc;
                        sourceTree = "<group>";
                };
+               7FE342442A0C3FA20015A058 /* coreml */ = {
+                       isa = PBXGroup;
+                       children = (
+                               7FE342452A0C3FA20015A058 /* whisper-encoder-impl.m */,
+                               7FE342462A0C3FA20015A058 /* whisper-encoder.h */,
+                               7FE342472A0C3FA20015A058 /* whisper-encoder.mm */,
+                               7FE342482A0C3FA20015A058 /* whisper-decoder-impl.h */,
+                               7FE342492A0C3FA20015A058 /* whisper-encoder-impl.h */,
+                               7FE3424A2A0C3FA20015A058 /* whisper-decoder-impl.m */,
+                       );
+                       name = coreml;
+                       path = ../../../coreml;
+                       sourceTree = "<group>";
+               };
 /* End PBXGroup section */
 
 /* Begin PBXNativeTarget section */
@@ -147,6 +174,7 @@
                        buildActionMask = 2147483647;
                        files = (
                                18627C8929052BE000BD2A04 /* LaunchScreen.storyboard in Resources */,
+                               7FE3424F2A0C418A0015A058 /* ggml-base.en-encoder.mlmodelc in Resources */,
                                18627C8629052BE000BD2A04 /* Assets.xcassets in Resources */,
                                18627C8429052BDF00BD2A04 /* Main.storyboard in Resources */,
                                18627C9B29052CFF00BD2A04 /* ggml-base.en.bin in Resources */,
@@ -161,11 +189,14 @@
                        buildActionMask = 2147483647;
                        files = (
                                18627C8129052BDF00BD2A04 /* ViewController.m in Sources */,
+                               7FE3424C2A0C3FA20015A058 /* whisper-encoder.mm in Sources */,
                                18627C9429052C4900BD2A04 /* whisper.cpp in Sources */,
                                18627C9629052C5800BD2A04 /* ggml.c in Sources */,
                                18627C7B29052BDF00BD2A04 /* AppDelegate.m in Sources */,
+                               7FE3424D2A0C3FA20015A058 /* whisper-decoder-impl.m in Sources */,
                                18627C8C29052BE000BD2A04 /* main.m in Sources */,
                                18627C7E29052BDF00BD2A04 /* SceneDelegate.m in Sources */,
+                               7FE3424B2A0C3FA20015A058 /* whisper-encoder-impl.m in Sources */,
                        );
                        runOnlyForDeploymentPostprocessing = 0;
                };