libvips · jcupitt · Aug 24, 2024 · Aug 22, 2024 · Aug 24, 2024 · Aug 24, 2024
diff --git a/ChangeLog b/ChangeLog
@@ -23,6 +23,7 @@ master 8.16
 - add vips_sdf(), vips_clamp(), vips_maxpair(), vips_minpair()
 - more const for the C++ API [Julianiolo]
 - deprecate "cache" (use tilecache instead)
+- add tests for tokenisation
 
 8.15.4
 

diff --git a/libvips/include/vips/util.h b/libvips/include/vips/util.h
@@ -379,6 +379,8 @@ typedef enum {
 	VIPS_TOKEN_COMMA
 } VipsToken;
 
+// we expose this one in the API for testing
+VIPS_API
 const char *vips__token_get(const char *buffer,
 	VipsToken *token, char *string, int size);
 const char *vips__token_must(const char *buffer, VipsToken *token,

diff --git a/test/meson.build b/test/meson.build
@@ -28,6 +28,23 @@ foreach script_test : script_tests
     )
 endforeach
 
+test_token = executable('test_token',
+    'test_token.c',
+    dependencies: libvips_dep,
+)
+
+test_token_sh = configure_file(
+    input: 'test_token.sh',
+    output: 'test_token.sh',
+    copy: true,
+)
+
+test('token',
+    test_token_sh,
+    depends: test_token,
+    workdir: meson.current_build_dir(),
+)
+
 test_connections = executable('test_connections',
     'test_connections.c',
     dependencies: libvips_dep,

diff --git a/test/test_token.c b/test/test_token.c
@@ -0,0 +1,59 @@
+/* compile with
+ *
+ * 		gcc -g -Wall token.c `pkg-config vips --cflags --libs`
+ *
+ * 	run with eg.:
+ *
+ * 		$ ./a.out '"wdfw"df,wdw,dw' 3 wdfw df,wdw,dw
+ *
+ */
+
+#include <vips/vips.h>
+
+char *token_names[] = {
+	"",
+	"left",
+	"right",
+	"string",
+	"equals",
+	"comma",
+};
+
+int
+main(int argc, char **argv)
+{
+	if (VIPS_INIT(argv[0]))
+		vips_error_exit(NULL);
+
+	if (argc != 5)
+		vips_error_exit("usage: %s string-to-parse token token-string residual",
+			argv[0]);
+
+	const char *p = argv[1];
+	printf("argv[1]:\n");
+	for (int i = 0; i < strlen(p); i++)
+		printf("\t\t%2d) %02x %c\n", i, p[i], p[i]);
+
+	VipsToken token;
+	char buf[256];
+	p = vips__token_get(p, &token, buf, 256);
+
+	printf("vips__token_get:\n");
+	printf("\ttoken = %d (%s)\n", token, token_names[token]);
+	if (token == VIPS_TOKEN_STRING) {
+		printf("\tbuf = <%s>\n", buf);
+		for (int i = 0; i < strlen(buf); i++)
+			printf("\t\t%2d) 0x%02x %c\n", i, buf[i], buf[i]);
+	}
+	printf("\tresidual = <%s>\n", p);
+
+	if (token != atoi(argv[2]))
+		vips_error_exit("token mismatch");
+	if (token == VIPS_TOKEN_STRING &&
+		!g_str_equal(buf, argv[3]))
+		vips_error_exit("parsed string mismatch");
+	if (!g_str_equal(p, argv[4]))
+		vips_error_exit("residual mismatch");
+
+	return 0;
+}
diff --git a/test/test_token.sh b/test/test_token.sh
@@ -0,0 +1,52 @@
+#!/bin/sh
+
+# test tokenisation
+
+# set -x
+# set -e
+
+# args are:
+#   string-to-parse
+#   expected-token
+#   expected-string (for string tokens)
+#   expected-residual
+check() {
+  test_name="$1"
+  test_string="$2"
+  residual="$3"
+  token="$4"
+  token_string="$5"
+
+  echo -n "$test_name ..."
+  log=$(./test_token "$2" "$3" "$4" "$5")
+  if [ $? -ne 0 ]; then
+    echo " FAIL"
+    echo "./test_token '$2' '$3' '$4' '$5'"
+    echo $log
+    exit 1
+  else
+    echo " yes"
+  fi
+}
+
+check "quoted strings end on the closing quote" \
+    '"ab"cd,abc,ab' 3 'ab' 'cd,abc,ab'
+
+check "quoted strings can have escaped quotes" \
+    '"ab\"cd",abc,ab' 3 'ab"cd' ',abc,ab'
+
+check "no closing quote" \
+    '"  abcd  ,abc,ab' 3 '  abcd  ,abc,ab' ''
+
+check "empty quote" \
+    '""abcd,abc,ab' 3 '' 'abcd,abc,ab'
+
+check "skip whitespace around quoted strings" \
+    '  "  abcd "  ,abc,ab' 3 '  abcd ' '  ,abc,ab'
+
+check "unquoted strings can have embedded quotes" \
+    'ab"cd,abc,ab' 3 'ab"cd' ',abc,ab'
+
+check "skip whitespace around unquoted strings" \
+    '  abcd  ,abc,ab' 3 'abcd' ',abc,ab'
+