From de10bd1d85c8012b6f9c07866c6fb2a18f5cb567 Mon Sep 17 00:00:00 2001 From: Michael Schmidt Date: Mon, 1 Jul 2019 19:55:12 +0200 Subject: [PATCH] Protobuf improvements (#1948) This makes numerous improvements to the Protobuf grammar adding (almost) full support for PB2 and PB3 syntax. --- components/prism-protobuf.js | 44 ++++++-- components/prism-protobuf.min.js | 2 +- examples/prism-protobuf.html | 25 +++++ .../protobuf/annotation_feature.test | 20 ++++ tests/languages/protobuf/builtin_feature.test | 39 +++++++ .../protobuf/class-name_feature.test | 102 ++++++++++++++++++ tests/languages/protobuf/keyword_feature.test | 63 +++++------ tests/languages/protobuf/map_feature.test | 23 ++++ tests/languages/protobuf/string_feature.test | 2 +- 9 files changed, 279 insertions(+), 41 deletions(-) create mode 100644 examples/prism-protobuf.html create mode 100644 tests/languages/protobuf/annotation_feature.test create mode 100644 tests/languages/protobuf/builtin_feature.test create mode 100644 tests/languages/protobuf/class-name_feature.test create mode 100644 tests/languages/protobuf/map_feature.test diff --git a/components/prism-protobuf.js b/components/prism-protobuf.js index 44e6cd5c..4e369f84 100644 --- a/components/prism-protobuf.js +++ b/components/prism-protobuf.js @@ -1,8 +1,36 @@ -Prism.languages.protobuf = Prism.languages.extend('clike', { - keyword: /\b(?:package|import|message|enum)\b/, - builtin: /\b(?:required|repeated|optional|reserved)\b/, - primitive: { - pattern: /\b(?:double|float|int32|int64|uint32|uint64|sint32|sint64|fixed32|fixed64|sfixed32|sfixed64|bool|string|bytes)\b/, - alias: 'symbol' - } -}); +(function (Prism) { + + var builtinTypes = /\b(?:double|float|[su]?int(?:32|64)|s?fixed(?:32|64)|bool|string|bytes)\b/; + + Prism.languages.protobuf = Prism.languages.extend('clike', { + 'class-name': { + pattern: /(\b(?:enum|extend|message|service)\s+)[A-Za-z_]\w*(?=\s*\{)/, + lookbehind: true + }, + 'keyword': /\b(?:enum|extend|extensions|import|message|oneof|option|optional|package|public|repeated|required|reserved|service|syntax|to)\b/ + }); + + Prism.languages.insertBefore('protobuf', 'operator', { + 'map': { + pattern: /\bmap<\s*[\w.]+\s*,\s*[\w.]+\s*>(?=\s+[A-Za-z_]\w*\s*[=;])/, + alias: 'class-name', + inside: { + 'punctuation': /[<>.,]/, + 'builtin': builtinTypes + } + }, + 'builtin': builtinTypes, + 'positional-class-name': { + pattern: /(?:\b|\B\.)[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*(?=\s+[A-Za-z_]\w*\s*[=;])/, + alias: 'class-name', + inside: { + 'punctuation': /\./ + } + }, + 'annotation': { + pattern: /(\[\s*)[A-Za-z_]\w*(?=\s*=)/, + lookbehind: true + } + }); + +}(Prism)); diff --git a/components/prism-protobuf.min.js b/components/prism-protobuf.min.js index a55dbe27..ffbdaee5 100644 --- a/components/prism-protobuf.min.js +++ b/components/prism-protobuf.min.js @@ -1 +1 @@ -Prism.languages.protobuf=Prism.languages.extend("clike",{keyword:/\b(?:package|import|message|enum)\b/,builtin:/\b(?:required|repeated|optional|reserved)\b/,primitive:{pattern:/\b(?:double|float|int32|int64|uint32|uint64|sint32|sint64|fixed32|fixed64|sfixed32|sfixed64|bool|string|bytes)\b/,alias:"symbol"}}); \ No newline at end of file +!function(e){var a=/\b(?:double|float|[su]?int(?:32|64)|s?fixed(?:32|64)|bool|string|bytes)\b/;e.languages.protobuf=e.languages.extend("clike",{"class-name":{pattern:/(\b(?:enum|extend|message|service)\s+)[A-Za-z_]\w*(?=\s*\{)/,lookbehind:!0},keyword:/\b(?:enum|extend|extensions|import|message|oneof|option|optional|package|public|repeated|required|reserved|service|syntax|to)\b/}),e.languages.insertBefore("protobuf","operator",{map:{pattern:/\bmap<\s*[\w.]+\s*,\s*[\w.]+\s*>(?=\s+[A-Za-z_]\w*\s*[=;])/,alias:"class-name",inside:{punctuation:/[<>.,]/,builtin:a}},builtin:a,"positional-class-name":{pattern:/(?:\b|\B\.)[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*(?=\s+[A-Za-z_]\w*\s*[=;])/,alias:"class-name",inside:{punctuation:/\./}},annotation:{pattern:/(\[\s*)[A-Za-z_]\w*(?=\s*=)/,lookbehind:!0}})}(Prism); \ No newline at end of file diff --git a/examples/prism-protobuf.html b/examples/prism-protobuf.html new file mode 100644 index 00000000..7174552d --- /dev/null +++ b/examples/prism-protobuf.html @@ -0,0 +1,25 @@ +

Full example

+
syntax = "proto3";
+
+package foo.generated;
+option java_package = "org.foo.generated";
+option optimize_for = SPEED;
+
+// What's up with all the foo?
+message Foo {
+
+  message Bar {
+
+    optional string key   = 1;
+    optional Foo value = 2;
+    optional string value_raw = 3 [deprecated=true];
+  }
+
+  enum Level {
+    INFO  = 0;
+    WARN  = 1;
+    ERROR = 2;
+  }
+
+  repeated Property property = 1;
+}
diff --git a/tests/languages/protobuf/annotation_feature.test b/tests/languages/protobuf/annotation_feature.test new file mode 100644 index 00000000..ec68a558 --- /dev/null +++ b/tests/languages/protobuf/annotation_feature.test @@ -0,0 +1,20 @@ +int32 foo = 1 [deprecated=true]; + +---------------------------------------------------- + +[ + ["builtin", "int32"], + " foo ", + ["operator", "="], + ["number", "1"], + ["punctuation", "["], + ["annotation", "deprecated"], + ["operator", "="], + ["boolean", "true"], + ["punctuation", "]"], + ["punctuation", ";"] +] + +---------------------------------------------------- + +Check for annotations. diff --git a/tests/languages/protobuf/builtin_feature.test b/tests/languages/protobuf/builtin_feature.test new file mode 100644 index 00000000..bf170b97 --- /dev/null +++ b/tests/languages/protobuf/builtin_feature.test @@ -0,0 +1,39 @@ +double +float +int32 +int64 +uint32 +uint64 +sint32 +sint64 +fixed32 +fixed64 +sfixed32 +sfixed64 +bool +string +bytes + +---------------------------------------------------- + +[ + ["builtin", "double"], + ["builtin", "float"], + ["builtin", "int32"], + ["builtin", "int64"], + ["builtin", "uint32"], + ["builtin", "uint64"], + ["builtin", "sint32"], + ["builtin", "sint64"], + ["builtin", "fixed32"], + ["builtin", "fixed64"], + ["builtin", "sfixed32"], + ["builtin", "sfixed64"], + ["builtin", "bool"], + ["builtin", "string"], + ["builtin", "bytes"] +] + +---------------------------------------------------- + +Check for builtin types. diff --git a/tests/languages/protobuf/class-name_feature.test b/tests/languages/protobuf/class-name_feature.test new file mode 100644 index 00000000..5f0ffb73 --- /dev/null +++ b/tests/languages/protobuf/class-name_feature.test @@ -0,0 +1,102 @@ +syntax = "proto2"; +syntax = "proto3"; + +option java_multiple_files = true; + +import public "new.proto"; +import "other.proto"; + +enum Foo {} +extend Foo {} +service Foo {} +message Foo { + Bar Bar = 0; + foo.Bar Bar2 = 0; + .baz.Bar Bar3 = 0; +} + +---------------------------------------------------- + +[ + ["keyword", "syntax"], + ["operator", "="], + ["string", "\"proto2\""], + ["punctuation", ";"], + + ["keyword", "syntax"], + ["operator", "="], + ["string", "\"proto3\""], + ["punctuation", ";"], + + + ["keyword", "option"], + " java_multiple_files ", + ["operator", "="], + ["boolean", "true"], + ["punctuation", ";"], + + + ["keyword", "import"], + ["keyword", "public"], + ["string", "\"new.proto\""], + ["punctuation", ";"], + + ["keyword", "import"], + ["string", "\"other.proto\""], + ["punctuation", ";"], + + + ["keyword", "enum"], + ["class-name", "Foo"], + ["punctuation", "{"], + ["punctuation", "}"], + + ["keyword", "extend"], + ["class-name", "Foo"], + ["punctuation", "{"], + ["punctuation", "}"], + + ["keyword", "service"], + ["class-name", "Foo"], + ["punctuation", "{"], + ["punctuation", "}"], + + ["keyword", "message"], + ["class-name", "Foo"], + ["punctuation", "{"], + + ["positional-class-name", [ + "Bar" + ]], + " Bar ", + ["operator", "="], + ["number", "0"], + ["punctuation", ";"], + + ["positional-class-name", [ + "foo", + ["punctuation", "."], + "Bar" + ]], + " Bar2 ", + ["operator", "="], + ["number", "0"], + ["punctuation", ";"], + + ["positional-class-name", [ + ["punctuation", "."], + "baz", + ["punctuation", "."], + "Bar" + ]], + " Bar3 ", + ["operator", "="], + ["number", "0"], + ["punctuation", ";"], + + ["punctuation", "}"] +] + +---------------------------------------------------- + +Check for class names diff --git a/tests/languages/protobuf/keyword_feature.test b/tests/languages/protobuf/keyword_feature.test index eabe71e6..1272833b 100644 --- a/tests/languages/protobuf/keyword_feature.test +++ b/tests/languages/protobuf/keyword_feature.test @@ -1,40 +1,41 @@ -message Point { - required int32 x = 1; - required int32 y = 2; - optional string label = 3; -} +enum +extend +extensions +import +message +oneof +option +optional +package +public +repeated +required +reserved +service +syntax +to ---------------------------------------------------- [ + ["keyword", "enum"], + ["keyword", "extend"], + ["keyword", "extensions"], + ["keyword", "import"], ["keyword", "message"], - " Point ", - ["punctuation", "{"], - - ["builtin", "required"], - ["primitive", "int32"], - " x ", - ["operator", "="], - ["number", "1"], - ["punctuation", ";"], - - ["builtin", "required"], - ["primitive", "int32"], - " y ", - ["operator", "="], - ["number", "2"], - ["punctuation", ";"], - - ["builtin", "optional"], - ["primitive", "string"], - " label ", - ["operator", "="], - ["number", "3"], - ["punctuation", ";"], - - ["punctuation", "}"] + ["keyword", "oneof"], + ["keyword", "option"], + ["keyword", "optional"], + ["keyword", "package"], + ["keyword", "public"], + ["keyword", "repeated"], + ["keyword", "required"], + ["keyword", "reserved"], + ["keyword", "service"], + ["keyword", "syntax"], + ["keyword", "to"] ] ---------------------------------------------------- -Check for keywords and builtins \ No newline at end of file +Check for keywords diff --git a/tests/languages/protobuf/map_feature.test b/tests/languages/protobuf/map_feature.test new file mode 100644 index 00000000..94a54f6c --- /dev/null +++ b/tests/languages/protobuf/map_feature.test @@ -0,0 +1,23 @@ +map bar; + +---------------------------------------------------- + +[ + ["map", [ + "map", + ["punctuation", "<"], + ["builtin", "string"], + ["punctuation", ","], + ["punctuation", "."], + "foo", + ["punctuation", "."], + "Foo", + ["punctuation", ">"] + ]], + " bar", + ["punctuation", ";"] +] + +---------------------------------------------------- + +Check for maps. diff --git a/tests/languages/protobuf/string_feature.test b/tests/languages/protobuf/string_feature.test index 3d2b45cb..89296969 100644 --- a/tests/languages/protobuf/string_feature.test +++ b/tests/languages/protobuf/string_feature.test @@ -20,4 +20,4 @@ ---------------------------------------------------- -Checks for single-quoted and double-quoted strings. \ No newline at end of file +Checks for single-quoted and double-quoted strings.