Merge pull request #783 from vsbogd/fix-newline

Fix newline
trueagi-io · Oct 14, 2024 · 0476b7d · 0476b7d
2 parents 1507f82 + 6bf24d3
commit 0476b7d
Show file tree

Hide file tree

Showing 4 changed files with 8 additions and 3 deletions.
diff --git a/Cargo.toml b/Cargo.toml
@@ -12,7 +12,7 @@ edition = "2021"
 
 [workspace.dependencies]
 hyperon = { path = "./lib", version = "0.2.0" }
-regex = "1.5.4"
+regex = "1.11.0"
 log = "0.4.0"
 env_logger = "0.8.4"
 

diff --git a/lib/src/metta/runner/stdlib_minimal.rs b/lib/src/metta/runner/stdlib_minimal.rs
@@ -506,7 +506,7 @@ pub fn register_rust_stdlib_tokens(target: &mut Tokenizer) {
         |token| { Ok(Atom::gnd(Number::from_float_str(token)?)) });
     tref.register_token(regex(r"True|False"),
         |token| { Atom::gnd(Bool::from_str(token)) });
-    tref.register_token(regex(r#"^".*"$"#),
+    tref.register_token(regex(r#"(?s)^".*"$"#),
         |token| { let mut s = String::from(token); s.remove(0); s.pop(); Atom::gnd(Str::from_string(s)) });
     let sum_op = Atom::gnd(SumOp{});
     tref.register_token(regex(r"\+"), move |_| { sum_op.clone() });
@@ -1285,13 +1285,17 @@ mod tests {
             !(id "te st")
             !(id "te\"st")
             !(id "")
+            !(id "te\nst")
+            !("te\nst"test)
         "#);
 
         assert_eq_metta_results!(metta.run(parser), Ok(vec![
             vec![expr!({Str::from_str("test")})],
             vec![expr!({Str::from_str("te st")})],
             vec![expr!({Str::from_str("te\"st")})],
             vec![expr!({Str::from_str("")})],
+            vec![expr!({Str::from_str("te\nst")})],
+            vec![expr!({Str::from_str("te\nst")} "test")],
         ]));
     }
 }
diff --git a/python/hyperon/stdlib.py b/python/hyperon/stdlib.py
@@ -119,7 +119,7 @@ def type_tokens():
         r"[-+]?\d+" : lambda token: ValueAtom(int(token), 'Number'),
         r"[-+]?\d+\.\d+": lambda token: ValueAtom(float(token), 'Number'),
         r"[-+]?\d+(\.\d+)?[eE][-+]?\d+": lambda token: ValueAtom(float(token), 'Number'),
-        r"^\".*\"$": lambda token: ValueAtom(str(token[1:-1]), 'String'),
+        r"(?s)^\".*\"$": lambda token: ValueAtom(str(token[1:-1]), 'String'),
         "\'[^\']\'": lambda token: ValueAtom(Char(token[1]), 'Char'),
         r"True|False": lambda token: ValueAtom(token == 'True', 'Bool'),
         r'regex:"[^"]*"': lambda token: G(RegexMatchableObject(token),  AtomType.UNDEFINED)

diff --git a/python/tests/test_stdlib.py b/python/tests/test_stdlib.py
@@ -79,6 +79,7 @@ def test_string_parsing(self):
         self.assertEqualMettaRunnerResults(metta.run("!(id' \"te st\")"), [[ValueAtom("te st")]])
         self.assertEqualMettaRunnerResults(metta.run("!(id' \"te\\\"st\")"), [[ValueAtom("te\"st")]])
         self.assertEqualMettaRunnerResults(metta.run("!(id' \"\")"), [[ValueAtom("")]])
+        self.assertEqualMettaRunnerResults(metta.run("!(id' \"te\\nst\")"), [[ValueAtom("te\nst")]])
 
     def test_regex(self):
         metta = MeTTa(env_builder=Environment.test_env())