From a4e75196c6a5fb554e79f8bed34c20d96256cb09 Mon Sep 17 00:00:00 2001 From: Dzuchun Date: Mon, 13 Jan 2025 03:53:31 +0200 Subject: [PATCH 1/6] Ignored proptest files --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index ea8c4bf..60f7a9c 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ /target +/proptest-regressions From 4bde45ef698e88d19e4b147ba1c4cdc737f3b6a0 Mon Sep 17 00:00:00 2001 From: Dzuchun Date: Mon, 13 Jan 2025 04:07:58 +0200 Subject: [PATCH 2/6] [Cargo.lock] --- Cargo.lock | 485 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 483 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b8c7db0..f09dc27 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,12 +2,39 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "autocfg" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + +[[package]] +name = "bitflags" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1be3f42a67d6d345ecd59f675f3f012d6974981560836e938c22b424b85ce1be" + [[package]] name = "byteorder" version = "1.5.0" @@ -20,6 +47,15 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", +] + [[package]] name = "derive_more" version = "1.0.0" @@ -41,6 +77,49 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "errno" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0d2fde1f7b3d48b8395d5f2de76c18a528bd6a9cdde438df747bfcba3e05d6f" + +[[package]] +name = "generic-array" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8c8444bc9d71b935156cc0ccab7f622180808af7867b1daae6547d773591703" +dependencies = [ + "typenum", +] + [[package]] name = "getrandom" version = "0.2.15" @@ -52,21 +131,56 @@ dependencies = [ "wasi", ] +[[package]] +name = "hashbrown" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "itoa" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + [[package]] name = "libc" version = "0.2.169" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + [[package]] name = "mathvis" version = "0.3.0" dependencies = [ "derive_more", + "generic-array", + "hashbrown", "nom", "num", + "paste", + "proptest", "rand", "thiserror", + "tracing", + "tracing-subscriber", ] [[package]] @@ -91,6 +205,16 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + [[package]] name = "num" version = "0.4.3" @@ -124,6 +248,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-integer" version = "0.1.46" @@ -164,6 +294,45 @@ dependencies = [ "autocfg", ] +[[package]] +name = "num_threads" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" +dependencies = [ + "libc", +] + +[[package]] +name = "once_cell" +version = "1.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" + +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.20" @@ -175,13 +344,39 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.92" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" +checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99" dependencies = [ "unicode-ident", ] +[[package]] +name = "proptest" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14cae93065090804185d3b75f0bf93b8eeda30c7a9b4a33d3bdb3988d6229e50" +dependencies = [ + "bit-set", + "bit-vec", + "bitflags", + "lazy_static", + "num-traits", + "rand", + "rand_chacha", + "rand_xorshift", + "regex-syntax", + "rusty-fork", + "tempfile", + "unarray", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + [[package]] name = "quote" version = "1.0.38" @@ -221,6 +416,75 @@ dependencies = [ "getrandom", ] +[[package]] +name = "rand_xorshift" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f" +dependencies = [ + "rand_core", +] + +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + +[[package]] +name = "rustix" +version = "0.38.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a78891ee6bf2340288408954ac787aa063d8e8817e9f53abb37c695c6d834ef6" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rusty-fork" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb3dcc6e454c328bb824492db107ab7c0ae8fcffe4ad210136ef014458c1bc4f" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] + +[[package]] +name = "serde" +version = "1.0.217" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.217" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + [[package]] name = "syn" version = "2.0.96" @@ -232,6 +496,20 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tempfile" +version = "3.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8a559c81686f576e8cd0290cd2a24a2a9ad80c98b3478856500fcbd7acd704" +dependencies = [ + "cfg-if", + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + [[package]] name = "thiserror" version = "2.0.11" @@ -252,6 +530,105 @@ dependencies = [ "syn", ] +[[package]] +name = "thread_local" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" +dependencies = [ + "cfg-if", + "once_cell", +] + +[[package]] +name = "time" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21" +dependencies = [ + "deranged", + "itoa", + "libc", + "num-conv", + "num_threads", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2834e6017e3e5e4b9834939793b282bc03b37a3336245fa820e35e233e2a85de" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tracing" +version = "0.1.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" +dependencies = [ + "once_cell", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" +dependencies = [ + "nu-ansi-term", + "sharded-slab", + "thread_local", + "time", + "tracing-core", +] + +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + [[package]] name = "unicode-ident" version = "1.0.14" @@ -264,12 +641,116 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "wait-timeout" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6" +dependencies = [ + "libc", +] + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + [[package]] name = "zerocopy" version = "0.7.35" From 376dd7ce1a7ed24c86aac6d7278e8bde590dfef9 Mon Sep 17 00:00:00 2001 From: Dzuchun Date: Mon, 13 Jan 2025 03:11:37 +0200 Subject: [PATCH 3/6] Implemented new lexing --- Cargo.lock | 30 -- Cargo.toml | 14 +- src/lexer.rs | 112 ++++++++ src/lexer_v2/mod.rs | 68 +++++ src/lexer_v2/number/mod.rs | 222 ++++++++++++++ src/lexer_v2/number/prop_tests.rs | 119 ++++++++ src/lexer_v2/number/tests.rs | 335 +++++++++++++++++++++ src/lexer_v2/prop_tests.rs | 17 ++ src/lexer_v2/token/mod.rs | 463 ++++++++++++++++++++++++++++++ src/lexer_v2/token/tests.rs | 164 +++++++++++ src/lexer_v2/word/mod.rs | 70 +++++ src/lexer_v2/word/prop_tests.rs | 35 +++ src/lexer_v2/word/tests.rs | 36 +++ src/lib.rs | 44 +++ 14 files changed, 1698 insertions(+), 31 deletions(-) create mode 100644 src/lexer_v2/mod.rs create mode 100644 src/lexer_v2/number/mod.rs create mode 100644 src/lexer_v2/number/prop_tests.rs create mode 100644 src/lexer_v2/number/tests.rs create mode 100644 src/lexer_v2/prop_tests.rs create mode 100644 src/lexer_v2/token/mod.rs create mode 100644 src/lexer_v2/token/tests.rs create mode 100644 src/lexer_v2/word/mod.rs create mode 100644 src/lexer_v2/word/prop_tests.rs create mode 100644 src/lexer_v2/word/tests.rs diff --git a/Cargo.lock b/Cargo.lock index f09dc27..a9223be 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,12 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "allocator-api2" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" - [[package]] name = "autocfg" version = "1.4.0" @@ -77,12 +71,6 @@ dependencies = [ "unicode-xid", ] -[[package]] -name = "equivalent" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" - [[package]] name = "errno" version = "0.3.10" @@ -105,12 +93,6 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "foldhash" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0d2fde1f7b3d48b8395d5f2de76c18a528bd6a9cdde438df747bfcba3e05d6f" - [[package]] name = "generic-array" version = "1.2.0" @@ -131,17 +113,6 @@ dependencies = [ "wasi", ] -[[package]] -name = "hashbrown" -version = "0.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" -dependencies = [ - "allocator-api2", - "equivalent", - "foldhash", -] - [[package]] name = "itoa" version = "1.0.14" @@ -172,7 +143,6 @@ version = "0.3.0" dependencies = [ "derive_more", "generic-array", - "hashbrown", "nom", "num", "paste", diff --git a/Cargo.toml b/Cargo.toml index 443f6b1..2247cea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,10 +6,22 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -derive_more = {version = "1.0.0", features = ["display"]} +derive_more = {version = "1.0.0", features = ["display", "as_ref", "deref"]} nom = "7.1.3" num = "0.4.3" thiserror = "2.0.11" +generic-array = "1.1.1" +tracing = { version = "0.1.41", default-features = false, features = ["attributes"] } [dev-dependencies] +paste = "1.0.15" rand = "0.8.5" +tracing-subscriber = { version = "0.3.19", default-features = false, features = ["std", "fmt", "time", "local-time", "ansi"] } +proptest = "1.6.0" + +[features] +default = ["std"] +std = [] + +[lints.rust] +unexpected_cfgs = { level = "warn", check-cfg = ['cfg(verbose_logging)'] } diff --git a/src/lexer.rs b/src/lexer.rs index 9221718..41dcc7f 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -11,6 +11,16 @@ use nom::{ sequence::{pair, preceded}, }; +use core::str::FromStr; + +use crate::{ + display_opt, + lexer_v2::{ + number::Number, + token::{SpecialKind, Token}, + }, +}; + type Res<'l, T = Token> = nom::IResult<&'l str, T, Error<&'l str>>; /// Represents type of grouping. @@ -46,6 +56,78 @@ pub enum Token { Identifier(String), } +<<<<<<< HEAD +======= +#[derive(Debug, Clone, Copy, thiserror::Error)] +#[error("Unsupported token: {_0}")] +pub struct UnsupportedToken<'s>(Token<&'s str, &'s str, &'s str>); + +impl<'s> TryFrom> for OldToken { + type Error = UnsupportedToken<'s>; + + fn try_from(value: Token<&'s str, &'s str, &'s str>) -> Result { + let token = match value { + Token::Grouping { + kind: crate::lexer_v2::token::GroupingKind::Parentheses, + side: crate::lexer_v2::token::GroupingSide::Left, + } => Self::GroupOpen(GroupingType::Parentheses), + Token::Grouping { + kind: crate::lexer_v2::token::GroupingKind::Brackets, + side: crate::lexer_v2::token::GroupingSide::Left, + } => Self::GroupOpen(GroupingType::Brackets), + Token::Grouping { + kind: crate::lexer_v2::token::GroupingKind::Braces, + side: crate::lexer_v2::token::GroupingSide::Left, + } => Self::GroupOpen(GroupingType::Braces), + Token::Grouping { + kind: crate::lexer_v2::token::GroupingKind::Parentheses, + side: crate::lexer_v2::token::GroupingSide::Right, + } => Self::GroupClose(GroupingType::Parentheses), + Token::Grouping { + kind: crate::lexer_v2::token::GroupingKind::Brackets, + side: crate::lexer_v2::token::GroupingSide::Right, + } => Self::GroupClose(GroupingType::Brackets), + Token::Grouping { + kind: crate::lexer_v2::token::GroupingKind::Braces, + side: crate::lexer_v2::token::GroupingSide::Right, + } => Self::GroupClose(GroupingType::Braces), + Token::Special(SpecialKind::Comma) => Self::Comma, + Token::Special(SpecialKind::Plus) => Self::Operator(Operator::Plus), + Token::Special(SpecialKind::Minus) => Self::Operator(Operator::Minus), + Token::Special(SpecialKind::Star) => Self::Operator(Operator::Star), + Token::Special(SpecialKind::Slash) => Self::Operator(Operator::Slash), + Token::Special(SpecialKind::Cap) => Self::Operator(Operator::Cap), + Token::Number(Number::Integer { int, exp }) => Self::Number( + f64::from_str(&format!("{int}.0{}", display_opt(&exp))) + .map_err(|_| UnsupportedToken(value))?, + ), + Token::Number(Number::Decimal { + int, + point, + frac, + exp, + }) => Self::Number( + f64::from_str(&format!("{int}{point}{frac}{}", display_opt(&exp))) + .map_err(|_| UnsupportedToken(value))?, + ), + Token::Number(Number::DecimalInt { int, point, exp }) => Self::Number( + f64::from_str(&format!("{int}{point}0{}", display_opt(&exp))) + .map_err(|_| UnsupportedToken(value))?, + ), + Token::Number(Number::DecimalFraction { point, frac, exp }) => Self::Number( + f64::from_str(&format!("0{point}{frac}{}", display_opt(&exp))) + .map_err(|_| UnsupportedToken(value))?, + ), + Token::Word(word) if *word == "i" => Self::ImaginaryUnit, + Token::Word(word) => Self::Identifier(word.to_string()), + _ => return Err(UnsupportedToken(value)), + }; + + Ok(token) + } +} + +>>>>>>> 7d0e9b0 (impl app) /// Represents token type (used for debug) #[derive(Debug, PartialEq, derive_more::Display)] pub enum TokenType { @@ -143,7 +225,15 @@ pub fn lex(i: &str) -> Res> { #[cfg(test)] mod tests { +<<<<<<< HEAD +======= + use alloc::borrow::ToOwned; + use alloc::vec::Vec; + +>>>>>>> 7d0e9b0 (impl app) use crate::lexer::GroupingType; + use crate::lexer_v2::token::Token; + use crate::lexer_v2::StrTokenParser; use super::lex; use super::Token; @@ -186,4 +276,26 @@ mod tests { )) ); } + + #[test] + fn tokens_new_to_old_same() { + crate::log_init(); + + // arrange + let input = "2 + x^2 - (3x^2 - 7y / sin[z])"; + + // act + let tokens = lex(input).expect("Must be a valid input").1; + let new_tokens = StrTokenParser::::new(input) + .inspect(|new_token| trace!(?new_token)) + .map(|p| { + p.expect("Should not encounter errors") + .try_into() + .expect("Should not have unsupported tokens") + }) + .collect::>(); + + // assert + assert_eq!(tokens, new_tokens); + } } diff --git a/src/lexer_v2/mod.rs b/src/lexer_v2/mod.rs new file mode 100644 index 0000000..2dbb9e9 --- /dev/null +++ b/src/lexer_v2/mod.rs @@ -0,0 +1,68 @@ +use core::marker::PhantomData; + +pub mod number; +pub mod token; +pub mod word; + +pub trait TokenInput { + fn is_empty(&self) -> bool; + fn on_invalid(self) -> Self; +} + +pub trait Parseable<'s>: Sized + 's { + fn try_parse(input: &'s str) -> Option<(Self, &'s str)>; +} + +impl<'s, T> StrTokenParser<'s, T> { + pub fn new(input: &'s str) -> Self { + Self { + input, + _phantom: PhantomData, + } + } +} + +pub struct StrTokenParser<'s, T> { + input: &'s str, + _phantom: PhantomData, +} + +fn skip_err(s: &str) -> (&str, &str) { + let next_whitespace = s + .char_indices() + .find_map(|(pos, ch)| ch.is_whitespace().then_some(pos)) + .unwrap_or(s.len()); + s.split_at(next_whitespace) +} + +impl<'s, T: Parseable<'s>> Iterator for StrTokenParser<'s, T> { + type Item = Result; + + #[tracing::instrument(level = "trace", skip(self), name = "token_parser", fields(input = self.input))] + fn next(&mut self) -> Option { + let ws = self.input.trim_start(); + trace!(ws); + if ws.is_empty() { + // no more tokens, exhausted the input + return None; + } + + if let Some((token, rest)) = T::try_parse(ws) { + // successful token parse, return the token + trace!("Parsed token"); + self.input = rest; + Some(Ok(token)) + } else { + // failed to parse the token + // + // skip to next whitespace character + let (erroneous, new_input) = skip_err(ws); + self.input = new_input; + trace!(erroneous); + Some(Err(erroneous)) + } + } +} + +#[cfg(test)] +mod prop_tests; diff --git a/src/lexer_v2/number/mod.rs b/src/lexer_v2/number/mod.rs new file mode 100644 index 0000000..c4d0a4a --- /dev/null +++ b/src/lexer_v2/number/mod.rs @@ -0,0 +1,222 @@ +use core::fmt::Display; +use std::ops::Not; + +use crate::display_opt; + +#[inline] +fn valid_sign(ch: char) -> bool { + matches!(ch, '+' | '-') +} + +#[inline] +fn valid_exponent_marker(ch: char) -> bool { + matches!(ch, 'e' | 'E') +} + +#[inline] +fn valid_decimal_point(ch: char) -> bool { + matches!(ch, '.') +} + +#[derive( + Debug, + Clone, + Copy, + Hash, + PartialEq, + Eq, + derive_more::AsRef, + derive_more::Deref, + derive_more::Display, +)] +#[display("{_0}")] +pub struct Digits(D); + +impl<'s> Digits<&'s str> { + #[tracing::instrument(level = "trace", ret, skip(input), fields(digits_input = input))] + fn parse(input: &'s str) -> Option<(Self, &'s str)> { + let end_pos = input + .char_indices() + .find_map(|(pos, ch)| ch.is_ascii_digit().not().then_some(pos)) + .unwrap_or(input.len()); + + if end_pos == 0 { + None + } else { + Some((Self(&input[..end_pos]), &input[end_pos..])) + } + } +} + +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, derive_more::Display)] +#[display("{e}{}{digits}", if let Some(s) = sign.as_ref() { s as &dyn Display } else { &"" as &dyn Display })] +pub struct Exponent { + e: C, + sign: Option, + digits: Digits, +} + +impl<'s> Exponent<&'s str, &'s str> { + #[tracing::instrument(level = "trace", ret, skip(input), fields(exponent_input = input))] + fn parse(input: &'s str) -> Option<(Self, &'s str)> { + let mut char_inds = input.char_indices(); + // if there's no first char, there's no exponent + let (_, e_marker) = char_inds.next()?; + if !valid_exponent_marker(e_marker) { + // does not start with an exponent marker, bail + return None; + } + + // exponent marker cannot go by itself + let (mut pos, maybe_sign) = char_inds.next()?; + let e = &input[..pos]; + // figure out sign + let sign; + if valid_sign(maybe_sign) { + // second char is a valid sign, query for next one + let (sign_end, _) = char_inds.next()?; + sign = Some(&input[pos..sign_end]); + pos = sign_end; + } else { + sign = None; + } + let (digits, rest) = Digits::parse(&input[pos..])?; + Some((Self { e, sign, digits }, rest)) + } +} + +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, derive_more::Display)] +pub enum Number { + #[display("int({int}e{})", display_opt(exp))] + Integer { + int: Digits, + exp: Option>, + }, + #[display("float({int}{point}{frac}{})", display_opt(exp))] + Decimal { + int: Digits, + point: C, + frac: Digits, + exp: Option>, + }, + #[display("float({int}{point}{})", display_opt(exp))] + DecimalInt { + int: Digits, + point: C, + // no frac part + exp: Option>, + }, + #[display("float({point}{})", display_opt(exp))] + DecimalFraction { + // no int part + point: C, + frac: Digits, + exp: Option>, + }, +} + +impl<'s> Number<&'s str, &'s str> { + #[tracing::instrument(level = "trace", skip(input), fields(number_input = input), ret)] + pub fn parse(mut input: &'s str) -> Option<(Self, &'s str)> { + let int_part: Option>; + + let frac_part: Option>; + let exp: Option>; + + if let Some((digits, rest)) = Digits::parse(input) { + int_part = Some(digits); + input = rest; + trace!(?int_part, input); + } else { + int_part = None; + } + + let point = 'point: { + // try parsing decimal point. if it's only a (optional) sign and integer part - it's a normal integer + let mut char_inds = input.char_indices(); + let Some((_, maybe_point)) = char_inds.next() else { + break 'point None; + }; + trace!(%maybe_point); + + // try parsing the sign + if !valid_decimal_point(maybe_point) { + break 'point None; + } + + // this is a valid point char, find there it ends + let point; + if let Some((point_end, _)) = char_inds.next() { + point = &input[..point_end]; + input = &input[point_end..]; + } else { + point = input; + input = ""; + }; + Some(point) + }; + trace!(?point, input); + + // try parsing fractional part + if let Some((digits, rest)) = Digits::parse(input) { + frac_part = Some(digits); + input = rest; + trace!(?frac_part, input); + } else { + frac_part = None; + } + + // try parsing exponent + if let Some((exponent, rest)) = Exponent::parse(input) { + exp = Some(exponent); + input = rest; + trace!(?exp, input); + } else { + exp = None; + } + + trace!(?int_part, ?point, ?frac_part); + match (int_part, point, frac_part) { + (Some(int), None, None) => Some((Self::Integer { int, exp }, input)), + (Some(int), Some(point), Some(frac)) => Some(( + Self::Decimal { + int, + point, + frac, + exp, + }, + input, + )), + (None, Some(point), Some(frac)) => { + Some((Self::DecimalFraction { point, frac, exp }, input)) + } + (Some(int), Some(point), None) => Some((Self::DecimalInt { int, point, exp }, input)), + _ => None, + } + } +} + +#[macro_export] +macro_rules! number { + (? $input:literal) => { + 'number: { + let Some((number, rest)) = $crate::lexer_v2::number::Number::parse(stringify!($input)) + else { + break 'number None; + }; + if !rest.trim().is_empty() { + break 'number None; + } + Some(number) + } + }; + ($input:literal) => { + number!(?$input).expect("Invalid number") + }; +} + +#[cfg(test)] +mod tests; + +#[cfg(test)] +mod prop_tests; diff --git a/src/lexer_v2/number/prop_tests.rs b/src/lexer_v2/number/prop_tests.rs new file mode 100644 index 0000000..54444ed --- /dev/null +++ b/src/lexer_v2/number/prop_tests.rs @@ -0,0 +1,119 @@ +use core::ops::Not; + +use super::{Digits, Exponent, Number}; + +use proptest::{prelude::ProptestConfig, prop_assert_eq, proptest}; + +proptest! { + #![proptest_config(ProptestConfig { + cases: 10_000, + max_default_size_range: 200, + ..ProptestConfig::default() + })] + #[test] + fn digits_never_panics(input: String) { + let _ = Digits::parse(&input); + } + + #[test] + fn digits_never_parse_letters(s in "[^0-9]+") { + prop_assert_eq!(Digits::parse(&s), None); + } + + #[test] + fn digits_always_parse_digits(digits in "[0-9]+", garbage in "[^0-9]+") { + let input = format!("{digits}{garbage}"); + prop_assert_eq!(Digits::parse(&input), Some((Digits(digits.as_str()), garbage.as_str()))); + } + + #[test] + fn exponent_never_panics(input: String) { + let _ = Exponent::parse(&input); + } + + #[test] + fn exponent_valid_parses(e in "[eE]", sign in "[+-]?", digits in "[0-9]+", sep in "[^0-9]", garbage: String) { + let input = format!("{e}{sign}{digits}{sep}{garbage}"); + let garbage = format!("{sep}{garbage}"); + prop_assert_eq!( + Exponent::parse(&input), + Some(( + Exponent{ + e: e.as_str(), + sign: sign.is_empty().not().then_some(sign.as_str()), + digits: Digits(digits.as_str()) + }, + garbage.as_str()) + )); + } + + #[test] + fn number_never_panics(input: String) { + let _ = Number::parse(&input); + } + + #[test] + fn number_parses_ints(digits in "[0-9]+", sep in "[^eE.0-9]", garbage: String) { + let input = format!("{digits}{sep}{garbage}"); + let garbage = format!("{sep}{garbage}"); + prop_assert_eq!( + Number::parse(&input), + Some((Number::Integer { int: Digits(digits.as_str()), exp: None }, garbage.as_str())) + ); + } + + #[test] + fn number_parses_doubles(mut part1 in "[0-9]+", mut part2 in "[0-9]*", swap_parts: bool, e in "[eE]", exponent_sign in "[+-]?", exponent in "[0-9]+", sep in "[^0-9]", garbage: String) { + if swap_parts { + core::mem::swap(&mut part1, &mut part2); + } + let input = format!("{part1}.{part2}{e}{exponent_sign}{exponent}{sep}{garbage}"); + let garbage = format!("{sep}{garbage}"); + let expected = match(part1.is_empty(), part2.is_empty()) { + (false, false) => Number::Decimal { + int: Digits(part1.as_str()), + point: ".", + frac: Digits(part2.as_str()), + exp: Some(Exponent { + e: e.as_str(), + sign: exponent_sign.is_empty().not().then_some(exponent_sign.as_str()), + digits: Digits(exponent.as_str()) + }) + }, + (true, false) => Number::DecimalFraction { + point: ".", + frac: Digits(part2.as_str()), + exp: Some(Exponent { + e: e.as_str(), + sign: exponent_sign.is_empty().not().then_some(exponent_sign.as_str()), + digits: Digits(exponent.as_str()) + }) + }, + (false, true) => Number::DecimalInt { + int: Digits(part1.as_str()), + point: ".", + exp: Some(Exponent { + e: e.as_str(), + sign: exponent_sign.is_empty().not().then_some(exponent_sign.as_str()), + digits: Digits(exponent.as_str()) + }) + }, + (true, true) => unreachable!(), + }; + prop_assert_eq!( + Number::parse(&input), + Some(( + expected, + garbage.as_str() + )) + ); + } + + #[test] + fn not_parses_garbage(input in "[^0-9.]") { + prop_assert_eq!( + Number::parse(&input), + None, + ); + } +} diff --git a/src/lexer_v2/number/tests.rs b/src/lexer_v2/number/tests.rs new file mode 100644 index 0000000..6895bb8 --- /dev/null +++ b/src/lexer_v2/number/tests.rs @@ -0,0 +1,335 @@ +use super::{Digits, Exponent, Number}; + +macro_rules! test_digits_parse { + ($name:ident, $input:literal, $expected:expr) => { + ::paste::paste! { + #[test] + fn [< $name _for_digits >]() { + $crate::log_init(); + + assert_eq!(Digits::parse($input), $expected); + } + } + }; +} + +test_digits_parse!(empty_should_fail, "", None); +test_digits_parse!(whitespace_should_fail, " ", None); +test_digits_parse!(letter_should_fail, "a", None); +test_digits_parse!(minus_should_fail, "-", None); +test_digits_parse!(minus_one_should_fail, "-1", None); + +test_digits_parse!(zero_should_ok, "0", Some((Digits("0"), ""))); +test_digits_parse!(p42_should_ok, "42", Some((Digits("42"), ""))); +test_digits_parse!( + number_space_should_leave_space, + "424233 ", + Some((Digits("424233"), " ")) +); +test_digits_parse!( + number_space_letter_should_leave_space_letter, + "424233 l", + Some((Digits("424233"), " l")) +); +test_digits_parse!( + number_letter_should_leave_letter, + "2p", + Some((Digits("2"), "p")) +); +test_digits_parse!( + big_number, + "9934023759839048593537636598927402740240740720909742094----bbb", + Some(( + Digits("9934023759839048593537636598927402740240740720909742094"), + "----bbb" + )) +); + +macro_rules! test_exponent_parse { + ($name:ident, $input:literal, $expected:expr) => { + ::paste::paste! { + #[test] + fn [< $name _for_exponent >]() { + $crate::log_init(); + + assert_eq!(Exponent::parse($input), $expected); + + } + } + }; +} + +test_exponent_parse!(empty_should_fail, "", None); +test_exponent_parse!(minus_should_fail, "-", None); +test_exponent_parse!(number_should_fail, "423", None); +test_exponent_parse!(only_e_should_fail, "e", None); +test_exponent_parse!(only_cap_e_should_fail, "E", None); + +test_exponent_parse!( + e1_should_ok, + "e1", + Some(( + Exponent { + e: "e", + sign: None, + digits: Digits("1"), + }, + "" + )) +); +test_exponent_parse!( + emore_should_ok, + "e92324l", + Some(( + Exponent { + e: "e", + sign: None, + digits: Digits("92324"), + }, + "l" + )) +); +test_exponent_parse!( + emore_minus_should_ok, + "e-92324l", + Some(( + Exponent { + e: "e", + sign: Some("-"), + digits: Digits("92324"), + }, + "l" + )) +); +test_exponent_parse!( + emore_plus_should_ok, + "e+92324l", + Some(( + Exponent { + e: "e", + sign: Some("+"), + digits: Digits("92324"), + }, + "l" + )) +); +test_exponent_parse!(emore_plus_plus_should_fail, "e++92324l", None); + +test_exponent_parse!( + emore_plus_num_plus_should_ok, + "E+92+", + Some(( + Exponent { + e: "E", + sign: Some("+"), + digits: Digits("92") + }, + "+" + )) +); + +macro_rules! test_number_parse { + ($name:ident, $input:literal, $expected:expr) => { + ::paste::paste! { + #[test] + fn [< $name _for_number >]() { + $crate::log_init(); + + assert_eq!(Number::parse($input), $expected); + } + } + }; +} + +test_number_parse!(empty_should_fail, "", None); +test_number_parse!(word_shold_fail, "word", None); + +test_number_parse!( + int_should_ok, + "1", + Some(( + Number::Integer { + int: Digits("1"), + exp: None + }, + "" + )) +); +test_number_parse!( + int_should_ok2, + "99", + Some(( + Number::Integer { + int: Digits("99"), + exp: None + }, + "" + )) +); + +test_number_parse!( + int_exp, + "99E2", + Some(( + Number::Integer { + int: Digits("99"), + exp: Some(Exponent { + e: "E", + sign: None, + digits: Digits("2") + }) + }, + "" + )) +); + +test_number_parse!( + int_exp_minus, + "99E-2", + Some(( + Number::Integer { + int: Digits("99"), + exp: Some(Exponent { + e: "E", + sign: Some("-"), + digits: Digits("2") + }) + }, + "" + )) +); +test_number_parse!( + int_exp_plus, + "99E+2", + Some(( + Number::Integer { + int: Digits("99"), + exp: Some(Exponent { + e: "E", + sign: Some("+"), + digits: Digits("2") + }) + }, + "" + )) +); +test_number_parse!( + int_plus_plus_should_no_exp, + "99E++2", + Some(( + Number::Integer { + int: Digits("99"), + exp: None + }, + "E++2" + )) +); + +test_number_parse!( + dec_int_only, + "2.", + Some(( + Number::DecimalInt { + int: Digits("2"), + point: ".", + exp: None + }, + "" + )) +); + +test_number_parse!( + dec_int_only_exp, + "2.E-3-", + Some(( + Number::DecimalInt { + int: Digits("2"), + point: ".", + exp: Some(Exponent { + e: "E", + sign: Some("-"), + digits: Digits("3") + }) + }, + "-" + )) +); + +test_number_parse!( + dec_int_full, + "3.141592--", + Some(( + Number::Decimal { + int: Digits("3"), + point: ".", + frac: Digits("141592"), + exp: None + }, + "--" + )) +); + +test_number_parse!( + dec_int_full_exp, + "3141.592e-3+3-", + Some(( + Number::Decimal { + int: Digits("3141"), + point: ".", + frac: Digits("592"), + exp: Some(Exponent { + e: "e", + sign: Some("-"), + digits: Digits("3") + }) + }, + "+3-" + )) +); + +test_number_parse!( + dec_int_frac_exp, + ".592e3+3-", + Some(( + Number::DecimalFraction { + point: ".", + frac: Digits("592"), + exp: Some(Exponent { + e: "e", + sign: None, + digits: Digits("3") + }) + }, + "+3-" + )) +); + +test_number_parse!( + dec_int_frac, + ".592_3+3-", + Some(( + Number::DecimalFraction { + point: ".", + frac: Digits("592"), + exp: None + }, + "_3+3-" + )) +); + +test_number_parse!( + dec_int_frac_exp2, + ".592E-37", + Some(( + Number::DecimalFraction { + point: ".", + frac: Digits("592"), + exp: Some(Exponent { + e: "E", + sign: Some("-"), + digits: Digits("37") + }) + }, + "" + )) +); diff --git a/src/lexer_v2/prop_tests.rs b/src/lexer_v2/prop_tests.rs new file mode 100644 index 0000000..d1ed0ff --- /dev/null +++ b/src/lexer_v2/prop_tests.rs @@ -0,0 +1,17 @@ +use super::{token::SpannedToken, StrTokenParser}; + +use proptest::{prelude::ProptestConfig, proptest}; + +proptest! { + #![proptest_config(ProptestConfig { + cases: 30_000, + max_default_size_range: 500, + ..ProptestConfig::default() + })] + + #[test] + fn parser_never_panics(input: String) { + let parser = StrTokenParser::<'_, SpannedToken<&str>>::new(&input); + let _ = parser.count(); + } +} diff --git a/src/lexer_v2/token/mod.rs b/src/lexer_v2/token/mod.rs new file mode 100644 index 0000000..791ca7a --- /dev/null +++ b/src/lexer_v2/token/mod.rs @@ -0,0 +1,463 @@ +use super::{number::Number, word::Word, Parseable}; + +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)] +pub struct SpannedToken { + token: Token, + span: Span, +} + +impl<'s> Parseable<'s> for SpannedToken<&'s str, &'s str, &'s str, &'s str> { + #[inline] + fn try_parse(input: &'s str) -> Option<(Self, &'s str)> { + Token::try_parse(input).map(|(token, rest)| { + ( + Self { + token, + span: &input[..input.len() - rest.len()], + }, + rest, + ) + }) + } +} + +/// Grouping symbol kind +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, derive_more::Display)] +pub enum GroupingKind { + #[display("paren")] + Parentheses, + #[display("bracket")] + Brackets, + #[display("brace")] + Braces, + #[display("angle")] + Angle, +} + +/// Grouping symbol kind +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, derive_more::Display)] +pub enum GroupingSide { + #[display("left")] + Left, + #[display("right")] + Right, +} + +/// Represents operators (not necessary corresponding to actual [`crate::evaluation_tree::Operator`]s). +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, derive_more::Display)] +pub enum SpecialKind { + #[display(".")] + Dot, + #[display(",")] + Comma, + #[display("+")] + Plus, + #[display("-")] + Minus, + #[display("*")] + Star, + #[display("/")] + Slash, + #[display("%")] + Percent, + #[display("^")] + Cap, + #[display("|")] + Pipe, + #[display("&")] + Umpersand, + #[display(":")] + Colon, + #[display("!")] + Bang, + #[display("=")] + Equals, + #[display("?")] + Question, + #[display("@")] + At, + #[display("~")] + Tilda, +} + +/// Represents a single token. +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, derive_more::Display)] +pub enum Token { + #[display("{kind}_{side}")] + Grouping { + kind: GroupingKind, + side: GroupingSide, + }, + #[display("{_0}")] + Special(SpecialKind), + #[display("{_0}")] + Number(Number), + #[display("{_0}")] + Word(Word), +} + +impl<'s> Parseable<'s> for Token<&'s str, &'s str, &'s str> { + fn try_parse(input: &'s str) -> Option<(Self, &'s str)> { + // if there's no char, there's no token + let c = input.chars().next()?; // (short-circuit) + + // try parsing word + // + // can't be confused with any other token, and mot of the tokens are expected to be words + if let Some((word, rest)) = Word::parse(input) { + return Some((Token::word(word), rest)); + } + + // try parsing number + // + // (same idea) + if let Some((number, rest)) = Number::parse(input) { + return Some((Token::number(number), rest)); + } + + // token is not a number and not a word, so it is a one-character token + let token = match c { + '(' => Token::left_paren(), + ')' => Token::right_paren(), + '[' => Token::left_bracket(), + ']' => Token::right_bracket(), + '{' => Token::left_brace(), + '}' => Token::right_brace(), + '.' => Token::dot(), + ',' => Token::comma(), + '+' => Token::plus(), + '-' => Token::minus(), + '*' => Token::star(), + '/' => Token::slash(), + '%' => Token::percent(), + '^' => Token::cap(), + '|' => Token::pipe(), + '&' => Token::umpersand(), + ':' => Token::colon(), + '!' => Token::bang(), + '=' => Token::equals(), + '?' => Token::question(), + '@' => Token::at(), + '~' => Token::tilda(), + // not a token + _ => return None, + }; + + let next_pos = input + .char_indices() + // position or the char after first + .nth(1) + // or end of the line, if that's the only char + .map_or(input.len(), |(pos, _)| pos); + + Some((token, &input[next_pos..])) + } +} + +// helper constructors and macro +impl Token { + pub fn dot() -> Self { + Token::Special(SpecialKind::Dot) + } + + pub fn comma() -> Self { + Token::Special(SpecialKind::Comma) + } + + pub fn left_paren() -> Self { + Self::Grouping { + kind: GroupingKind::Parentheses, + side: GroupingSide::Left, + } + } + + pub fn right_paren() -> Self { + Self::Grouping { + kind: GroupingKind::Parentheses, + side: GroupingSide::Right, + } + } + + pub fn left_bracket() -> Self { + Self::Grouping { + kind: GroupingKind::Brackets, + side: GroupingSide::Left, + } + } + + pub fn right_bracket() -> Self { + Self::Grouping { + kind: GroupingKind::Brackets, + side: GroupingSide::Right, + } + } + + pub fn left_brace() -> Self { + Self::Grouping { + kind: GroupingKind::Braces, + side: GroupingSide::Left, + } + } + + pub fn right_brace() -> Self { + Self::Grouping { + kind: GroupingKind::Braces, + side: GroupingSide::Right, + } + } + + pub fn left_angle() -> Self { + Self::Grouping { + kind: GroupingKind::Angle, + side: GroupingSide::Left, + } + } + + pub fn right_angle() -> Self { + Self::Grouping { + kind: GroupingKind::Angle, + side: GroupingSide::Right, + } + } + + pub fn plus() -> Self { + Token::Special(SpecialKind::Plus) + } + + pub fn minus() -> Self { + Token::Special(SpecialKind::Minus) + } + + pub fn star() -> Self { + Token::Special(SpecialKind::Star) + } + + pub fn slash() -> Self { + Token::Special(SpecialKind::Slash) + } + + pub fn percent() -> Self { + Token::Special(SpecialKind::Percent) + } + + pub fn cap() -> Self { + Token::Special(SpecialKind::Cap) + } + + pub fn pipe() -> Self { + Token::Special(SpecialKind::Pipe) + } + + pub fn umpersand() -> Self { + Token::Special(SpecialKind::Umpersand) + } + + pub fn colon() -> Self { + Token::Special(SpecialKind::Colon) + } + + pub fn bang() -> Self { + Token::Special(SpecialKind::Bang) + } + + pub fn equals() -> Self { + Token::Special(SpecialKind::Equals) + } + + pub fn question() -> Self { + Token::Special(SpecialKind::Question) + } + + pub fn at() -> Self { + Token::Special(SpecialKind::At) + } + + pub fn tilda() -> Self { + Token::Special(SpecialKind::Tilda) + } + + pub fn number(n: Number) -> Self { + Token::Number(n) + } + + pub fn word(w: Word) -> Self { + Token::Word(w) + } +} + +#[macro_export] +macro_rules! tokens_inner { + () => { + arr![] + }; + ( ( $($inner:tt)* ) $($rest:tt)* ) => { + Concat::concat( + Concat::concat( + Concat::concat( + arr![Token::left_paren()], + tokens_inner!($($inner)*) + ), + arr![Token::right_paren()], + ), + tokens_inner!($($rest)*) + ) + }; + ( [ $($inner:tt)* ] $($rest:tt)* ) => { + Concat::concat( + Concat::concat( + Concat::concat( + arr![Token::left_bracket()], + tokens_inner!($($inner)*) + ), + arr![Token::right_bracket()], + ), + tokens_inner!($($rest)*) + ) + }; + ( { $($inner:tt)* } $($rest:tt)* ) => { + Concat::concat( + Concat::concat( + Concat::concat( + arr![Token::left_brace()], + tokens_inner!($($inner)*) + ), + arr![Token::right_brace()], + ), + tokens_inner!($($rest)*) + ) + }; + (. $($rest:tt)*) => { + Concat::concat( + arr![Token::dot()], + tokens_inner!($($rest)*) + ) + }; + (, $($rest:tt)*) => { + Concat::concat( + arr![Token::comma()], + tokens_inner!($($rest)*) + ) + }; + (+ $($rest:tt)*) => { + Concat::concat( + arr![Token::plus()], + tokens_inner!($($rest)*) + ) + }; + (- $($rest:tt)*) => { + Concat::concat( + arr![Token::minus()], + tokens_inner!($($rest)*) + ) + }; + (* $($rest:tt)*) => { + Concat::concat( + arr![Token::star()], + tokens_inner!($($rest)*) + ) + }; + (/ $($rest:tt)*) => { + Concat::concat( + arr![Token::slash()], + tokens_inner!($($rest)*) + ) + }; + (% $($rest:tt)*) => { + Concat::concat( + arr![Token::percent()], + tokens_inner!($($rest)*) + ) + }; + (^ $($rest:tt)*) => { + Concat::concat( + arr![Token::cap()], + tokens_inner!($($rest)*) + ) + }; + (| $($rest:tt)*) => { + Concat::concat( + arr![Token::pipe()], + tokens_inner!($($rest)*) + ) + }; + (& $($rest:tt)*) => { + Concat::concat( + arr![Token::umpersand()], + tokens_inner!($($rest)*) + ) + }; + (: $($rest:tt)*) => { + Concat::concat( + arr![Token::colon()], + tokens_inner!($($rest)*) + ) + }; + (! $($rest:tt)*) => { + Concat::concat( + arr![Token::bang()], + tokens_inner!($($rest)*) + ) + }; + (= $($rest:tt)*) => { + Concat::concat( + arr![Token::equals()], + tokens_inner!($($rest)*) + ) + }; + (? $($rest:tt)*) => { + Concat::concat( + arr![Token::question()], + tokens_inner!($($rest)*) + ) + }; + (@ $($rest:tt)*) => { + Concat::concat( + arr![Token::at()], + tokens_inner!($($rest)*) + ) + }; + (~ $($rest:tt)*) => { + Concat::concat( + arr![Token::tilda()], + tokens_inner!($($rest)*) + ) + }; + (< $($rest:tt)*) => { + Concat::concat( + arr![Token::left_angle()], + tokens_inner!($($rest)*) + ) + }; + (> $($rest:tt)*) => { + Concat::concat( + arr![Token::right_angle()], + tokens_inner!($($rest)*) + ) + }; + ($token:ident $($rest:tt)*) => { + Concat::concat( + arr![Token::word(word!($token))], + tokens_inner!($($rest)*) + ) + }; + ($token:literal $($rest:tt)*) => { + Concat::concat( + arr![Token::number(number!($token))], + tokens_inner!($($rest)*) + ) + }; +} + +#[macro_export] +macro_rules! tokens { + ($($rest:tt)*) => {{ + #[allow(unused_imports)] + use generic_array::{arr, sequence::Concat}; + #[allow(unused_imports)] + use $crate::{lexer_v2::token::Token, word, tokens_inner, number}; + tokens_inner!($($rest)*) + }}; +} + +#[cfg(test)] +mod tests; diff --git a/src/lexer_v2/token/tests.rs b/src/lexer_v2/token/tests.rs new file mode 100644 index 0000000..a3b4a3d --- /dev/null +++ b/src/lexer_v2/token/tests.rs @@ -0,0 +1,164 @@ +use super::super::StrTokenParser; +use super::Token; +use crate::{number, word}; + +macro_rules! test_tokens { + ($(#[$($attr:tt)+])* $name:ident, ($($input:tt)*), [$($expected:expr),*]) => { + ::paste::paste! { + #[test] + $( + #[$($attr)*] + )* + fn [< $name _tokens >]() { + $crate::log_init(); + + let input = tokens!($($input)*); + let expected = generic_array::GenericArray::, _>::from_array([$($expected),*]); + + assert_eq!(input, expected); + } + } + }; +} + +test_tokens!(empty, (), []); + +test_tokens!(dot, (.), [Token::dot()]); +test_tokens!(comma, (,), [Token::comma()]); +test_tokens!(plus, (+), [Token::plus()]); +test_tokens!(minus, (-), [Token::minus()]); +test_tokens!(star, (*), [Token::star()]); +test_tokens!(slash, (/), [Token::slash()]); +test_tokens!(percent, (%), [Token::percent()]); +test_tokens!(cap, (^), [Token::cap()]); +test_tokens!(pipe, (|), [Token::pipe()]); +test_tokens!(umpersand, (&), [Token::umpersand()]); +test_tokens!(colon, (:), [Token::colon()]); +test_tokens!(bang, (!), [Token::bang()]); +test_tokens!(equals, (=), [Token::equals()]); +test_tokens!(question, (?), [Token::question()]); +test_tokens!(at, (@), [Token::at()]); +test_tokens!(tilda, (~), [Token::tilda()]); +test_tokens!(left_angle, (<), [Token::left_angle()]); +test_tokens!(right_angle, (>), [Token::right_angle()]); + +test_tokens!(word, (word), [Token::word(word!(word))]); +test_tokens!(word2, (w0r3d), [Token::word(word!(w0r3d))]); +test_tokens!( + long_word, + (SOME_REALLY_LONG_WORD_1111111), + [Token::word(word!(SOME_REALLY_LONG_WORD_1111111))] +); +test_tokens!( + #[should_panic = "Invalid word"] + invalid_word, + (r#not_a_valid_word), + [Token::word(word!(not_a_valid_word))] +); + +test_tokens!(number, (1), [Token::number(number!(1))]); +test_tokens!(number2, (1.), [Token::number(number!(1.))]); +test_tokens!(number3, (1.), [Token::number(number!(1.))]); +test_tokens!(number4, (1.0E0), [Token::number(number!(1.0E0))]); + +test_tokens!( + negative_number, + (-24.003E6), + [Token::minus(), Token::number(number!(24.003E6))] +); + +test_tokens!( + very_much_positive_number, + (+ + + 42.0E-5), + [Token::plus(), Token::plus(), Token::plus(), Token::number(number!(42.0E-5))] +); + +test_tokens!( + random_expr, + (3 + 4 - 6 + twelve - 00 - ONLY), + [ + Token::number(number!(3)), + Token::plus(), + Token::number(number!(4)), + Token::minus(), + Token::number(number!(6)), + Token::plus(), + Token::word(word!(twelve)), + Token::minus(), + Token::number(number!(00)), + Token::minus(), + Token::word(word!(ONLY)) + ] +); + +test_tokens!( + parens, + ((2 + 3.4e-21) | [a - _BB45] {} / 10), + [ + Token::left_paren(), + Token::number(number!(2)), + Token::plus(), + Token::number(number!(3.4e-21)), + Token::right_paren(), + Token::pipe(), + Token::left_bracket(), + Token::word(word!(a)), + Token::minus(), + Token::word(word!(_BB45)), + Token::right_bracket(), + Token::left_brace(), + Token::right_brace(), + Token::slash(), + Token::number(number!(10)) + ] +); + +macro_rules! test_token_parser { + ($name:ident, $input:literal, [$($token:expr),*$(,)?]) => { + ::paste::paste! { + #[test] + fn [< $name _tokens_iter >]() { + $crate::log_init(); + + let iter = StrTokenParser::<'_, Token>::new($input); + + let parsed_tokens = iter.collect::>(); + + assert_eq!(parsed_tokens.as_slice(), &[$($token),*]); + } + } + }; +} + +test_token_parser!(empty_parses, "", []); +test_token_parser!(plus_pares, "+", [Ok(Token::plus())]); +test_token_parser!( + expr_parses, + "+3 - b4_A[AA/ #iim 3e-67+ (17.00E-67c ]}- an mee#34\nb) \t ---", + [ + Ok(Token::plus()), + Ok(Token::number(number!(3))), + Ok(Token::minus()), + Ok(Token::word(word!(b4_A))), + Ok(Token::left_bracket()), + Ok(Token::word(word!(AA))), + Ok(Token::slash()), + Err("#iim"), + Ok(Token::number(number!(3e-67))), + Ok(Token::plus()), + Ok(Token::left_paren()), + Ok(Token::number(number!(17.00E-67))), + Ok(Token::word(word!(c))), + Ok(Token::right_bracket()), + Ok(Token::right_brace()), + Ok(Token::minus()), + Ok(Token::word(word!(an))), + Ok(Token::word(word!(mee))), + Err("#34"), + Ok(Token::word(word!(b))), + Ok(Token::right_paren()), + Ok(Token::minus()), + Ok(Token::minus()), + Ok(Token::minus()), + ] +); diff --git a/src/lexer_v2/word/mod.rs b/src/lexer_v2/word/mod.rs new file mode 100644 index 0000000..2f4300b --- /dev/null +++ b/src/lexer_v2/word/mod.rs @@ -0,0 +1,70 @@ +use std::ops::Not; + +#[derive( + Debug, + Clone, + Copy, + Hash, + PartialEq, + Eq, + derive_more::AsRef, + derive_more::Deref, + derive_more::Display, +)] +#[display("{_0}")] +pub struct Word(W); + +#[inline] +fn valid_start(ch: char) -> bool { + matches!(ch, 'a'..='z' | 'A'..='Z' | '_') +} + +#[inline] +fn valid_continue(ch: char) -> bool { + matches!(ch, 'a'..='z' | 'A'..='Z' | '_' | '0'..='9') +} + +impl<'s> Word<&'s str> { + pub fn parse(input: &'s str) -> Option<(Self, &'s str)> { + let mut char_inds = input + // be a good (cat)boy and iterate over characters + .char_indices(); + // get first non-whitespace char (if there's none, we are done) + let (_, first_char) = char_inds.next()?; + if !valid_start(first_char) { + // first non-whitespace is not a valid started for word; bail + return None; + } + // word will end, once we + let end_pos = char_inds + // find next first char that is not a valid continue + .find_map(|(pos, ch)| valid_continue(ch).not().then_some(pos)) + // if there none, word is up to the end of string + .unwrap_or(input.len()); + Some((Self(&input[..end_pos]), &input[end_pos..])) + } +} + +#[macro_export] +macro_rules! word { + (? $input:ident) => { + 'word: { + let Some((word, rest)) = $crate::lexer_v2::word::Word::parse(stringify!($input)) else { + break 'word None; + }; + if !rest.trim().is_empty() { + break 'word None; + } + Some(word) + } + }; + ($input:ident) => { + word!(?$input).expect("Invalid word") + }; +} + +#[cfg(test)] +mod tests; + +#[cfg(test)] +mod prop_tests; diff --git a/src/lexer_v2/word/prop_tests.rs b/src/lexer_v2/word/prop_tests.rs new file mode 100644 index 0000000..8fecec5 --- /dev/null +++ b/src/lexer_v2/word/prop_tests.rs @@ -0,0 +1,35 @@ +use super::Word; + +use proptest::{prelude::ProptestConfig, prop_assert_eq, proptest}; + +proptest! { + #![proptest_config(ProptestConfig { + cases: 10_000, + max_default_size_range: 200, + ..ProptestConfig::default() + })] + + #[test] + fn never_panics(input: String) { + let _ = Word::parse(&input); + } + + #[test] + fn parses_words(word in "[a-zA-Z_][a-zA-Z0-9_]*", sep in "[^a-zA-Z0-9_]", garbage: String) { + let input = format!("{word}{sep}{garbage}"); + let garbage = format!("{sep}{garbage}"); + + prop_assert_eq!( + Word::parse(&input), + Some((Word(word.as_str()), garbage.as_str())) + ); + } + + #[test] + fn not_parses_garbage(input in "[^a-zA-Z_]*") { + prop_assert_eq!( + Word::parse(&input), + None, + ); + } +} diff --git a/src/lexer_v2/word/tests.rs b/src/lexer_v2/word/tests.rs new file mode 100644 index 0000000..ed80e19 --- /dev/null +++ b/src/lexer_v2/word/tests.rs @@ -0,0 +1,36 @@ +use super::Word; + +macro_rules! test_word_parse { + ($name:ident, $input:literal, $expected:expr) => { + #[test] + fn $name() { + assert_eq!(Word::parse($input), $expected); + } + }; +} + +test_word_parse!(empty_should_fail, "", None); +test_word_parse!(number_should_fail, "1", None); +test_word_parse!(whitespace_should_fail, " ", None); +test_word_parse!(non_ascii_should_fail, "слово", None); +test_word_parse!(minus_should_fail, "-", None); + +test_word_parse!(letter_should_ok, "a", Some((Word("a"), ""))); +test_word_parse!(letter_should_ok2, "U", Some((Word("U"), ""))); +test_word_parse!(underscore_should_ok, "_", Some((Word("_"), ""))); +test_word_parse!( + letter_space_should_leave_space, + "a ", + Some((Word("a"), " ")) +); +test_word_parse!( + letter_something_should_keep_something, + "a something", + Some((Word("a"), " something")) +); + +test_word_parse!( + a_minus_b_should_leave_minus_b, + "a-b", + Some((Word("a"), "-b")) +); diff --git a/src/lib.rs b/src/lib.rs index fbe8e80..ebeb518 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,50 @@ #![doc = include_str!("../README.md")] +use core::fmt::Display; + pub mod evaluation_tree; pub mod lexer; pub mod parser; +pub mod lexer_v2; + +#[macro_use] +extern crate tracing; + +#[cfg(test)] +#[allow(unused)] +pub(crate) fn log_init() { + use tracing::level_filters::LevelFilter; + use tracing_subscriber::fmt::{self, format::FmtSpan, time::OffsetTime}; + + #[cfg(verbose_logging)] + if fmt::fmt() + .with_max_level(LevelFilter::TRACE) + .with_span_events(FmtSpan::ACTIVE) + .with_line_number(true) + .with_level(true) + .with_target(true) + .with_timer(OffsetTime::local_rfc_3339().expect("Failed to get local timer")) + .compact() + .try_init() + .is_ok() + { + info!("Log up!"); + } else { + eprintln!("Log is up?"); + } +} + +#[allow( + clippy::ref_option, + reason = "Use case. Don't want to call `.as_ref()` on each site" +)] +pub(crate) fn display_opt(t: &Option) -> &dyn Display { + if let Some(t) = t.as_ref() { + t as &dyn Display + } else { + &"" as &dyn Display + } +} +#[cfg(test)] +extern crate alloc; From d70e25cc5ecaa432aef791087465072d4c129f0a Mon Sep 17 00:00:00 2001 From: Dzuchun Date: Mon, 13 Jan 2025 03:12:17 +0200 Subject: [PATCH 4/6] Made `std` optional. --- Cargo.lock | 30 ++++++++++++++++++++++++++++++ Cargo.toml | 4 ++-- src/evaluation_tree/args.rs | 14 ++++++++------ src/evaluation_tree/arithmetic.rs | 2 ++ src/evaluation_tree/functions.rs | 1 + src/evaluation_tree/mod.rs | 2 ++ src/lexer.rs | 11 ++++++++--- src/lexer_v2/number/mod.rs | 3 +-- src/lexer_v2/number/prop_tests.rs | 2 ++ src/lexer_v2/prop_tests.rs | 2 ++ src/lexer_v2/token/tests.rs | 2 ++ src/lexer_v2/word/mod.rs | 2 +- src/lexer_v2/word/prop_tests.rs | 2 ++ src/lib.rs | 8 +++++++- src/parser.rs | 9 ++++++++- 15 files changed, 78 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a9223be..f09dc27 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "autocfg" version = "1.4.0" @@ -71,6 +77,12 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "errno" version = "0.3.10" @@ -93,6 +105,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0d2fde1f7b3d48b8395d5f2de76c18a528bd6a9cdde438df747bfcba3e05d6f" + [[package]] name = "generic-array" version = "1.2.0" @@ -113,6 +131,17 @@ dependencies = [ "wasi", ] +[[package]] +name = "hashbrown" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + [[package]] name = "itoa" version = "1.0.14" @@ -143,6 +172,7 @@ version = "0.3.0" dependencies = [ "derive_more", "generic-array", + "hashbrown", "nom", "num", "paste", diff --git a/Cargo.toml b/Cargo.toml index 2247cea..b790378 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,7 @@ num = "0.4.3" thiserror = "2.0.11" generic-array = "1.1.1" tracing = { version = "0.1.41", default-features = false, features = ["attributes"] } +hashbrown = "0.15.2" [dev-dependencies] paste = "1.0.15" @@ -20,8 +21,7 @@ tracing-subscriber = { version = "0.3.19", default-features = false, features = proptest = "1.6.0" [features] -default = ["std"] -std = [] +default = [] [lints.rust] unexpected_cfgs = { level = "warn", check-cfg = ['cfg(verbose_logging)'] } diff --git a/src/evaluation_tree/args.rs b/src/evaluation_tree/args.rs index 2ed55a1..68cbfb7 100644 --- a/src/evaluation_tree/args.rs +++ b/src/evaluation_tree/args.rs @@ -16,14 +16,16 @@ //! //! Variables/functions can be unregistered too. Although, there's not much use for it apart from debugging. -use std::{ - collections::{HashMap, HashSet}, - fmt::Debug, -}; +use core::fmt::Debug; + +use alloc::{borrow::ToOwned, boxed::Box, string::String, vec::Vec}; use num::complex::Complex64; + use thiserror::Error; +use hashbrown::{HashMap, HashSet}; + /// Represents types of arguments. /// /// - [`ArgType::Variable`] stands for regular complex variable.s @@ -75,7 +77,7 @@ impl AsRef for ArgsErased { pub struct ArgsRefErased<'l>(pub &'l Args); impl Debug for ArgsRefErased<'_> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { fn display_map(map: &HashMap) -> String { format!( "{{{}}}", @@ -94,7 +96,7 @@ impl Debug for ArgsRefErased<'_> { } impl Debug for ArgsErased { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { ArgsRefErased(&self.0).fmt(f) } } diff --git a/src/evaluation_tree/arithmetic.rs b/src/evaluation_tree/arithmetic.rs index 4ad079e..bb5532a 100644 --- a/src/evaluation_tree/arithmetic.rs +++ b/src/evaluation_tree/arithmetic.rs @@ -27,6 +27,7 @@ impl Operator for Negation { #[cfg(test)] mod negation_tests { + use alloc::borrow::ToOwned; use num::complex::Complex64; use crate::evaluation_tree::{ @@ -111,6 +112,7 @@ impl Operator for Addition { #[cfg(test)] mod addition_tests { + use alloc::borrow::ToOwned; use num::complex::Complex64; use crate::evaluation_tree::{ diff --git a/src/evaluation_tree/functions.rs b/src/evaluation_tree/functions.rs index 512b3a1..c0e9b5d 100644 --- a/src/evaluation_tree/functions.rs +++ b/src/evaluation_tree/functions.rs @@ -6,6 +6,7 @@ //! //! In presented API, only [`NamedFunction`]s are used for now. Although, I might add lots of default functions that will automatically resolve to their [`Functional`] counterparts during parsing. +use alloc::string::String; use num::complex::Complex64; use super::{ diff --git a/src/evaluation_tree/mod.rs b/src/evaluation_tree/mod.rs index 938ad2f..0f334c7 100644 --- a/src/evaluation_tree/mod.rs +++ b/src/evaluation_tree/mod.rs @@ -6,6 +6,7 @@ //! [`EvaluationTree`] can be created by parsing with [`EvaluationTree::from_tokens`] function. //! If you need some custom structure, [`EvaluationTree`] can be ignored and bare [`Node`]s used instead, as they both implement [`Evaluatable`] anyway. +use alloc::{boxed::Box, string::String}; use num::{complex::Complex64, Complex}; use crate::{ @@ -182,6 +183,7 @@ where #[cfg(test)] mod tests { + use alloc::borrow::ToOwned; use num::complex::Complex64; use crate::evaluation_tree::args::{Args, MissingError}; diff --git a/src/lexer.rs b/src/lexer.rs index 41dcc7f..8f02d10 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -1,5 +1,10 @@ //! This module defines lexer and structs relevant to it. +use alloc::{ + borrow::ToOwned, + string::{String, ToString}, + vec::Vec, +}; use nom::{ branch::alt, bytes::complete::tag, @@ -11,8 +16,6 @@ use nom::{ sequence::{pair, preceded}, }; -use core::str::FromStr; - use crate::{ display_opt, lexer_v2::{ @@ -21,7 +24,9 @@ use crate::{ }, }; -type Res<'l, T = Token> = nom::IResult<&'l str, T, Error<&'l str>>; +use core::{fmt::Display, str::FromStr}; + +type Res<'l, T = OldToken> = nom::IResult<&'l str, T, Error<&'l str>>; /// Represents type of grouping. #[derive(Debug, PartialEq, Clone)] diff --git a/src/lexer_v2/number/mod.rs b/src/lexer_v2/number/mod.rs index c4d0a4a..1460bdb 100644 --- a/src/lexer_v2/number/mod.rs +++ b/src/lexer_v2/number/mod.rs @@ -1,5 +1,4 @@ -use core::fmt::Display; -use std::ops::Not; +use core::{fmt::Display, ops::Not}; use crate::display_opt; diff --git a/src/lexer_v2/number/prop_tests.rs b/src/lexer_v2/number/prop_tests.rs index 54444ed..4e14b2b 100644 --- a/src/lexer_v2/number/prop_tests.rs +++ b/src/lexer_v2/number/prop_tests.rs @@ -4,6 +4,8 @@ use super::{Digits, Exponent, Number}; use proptest::{prelude::ProptestConfig, prop_assert_eq, proptest}; +use alloc::string::String; + proptest! { #![proptest_config(ProptestConfig { cases: 10_000, diff --git a/src/lexer_v2/prop_tests.rs b/src/lexer_v2/prop_tests.rs index d1ed0ff..f42ffaf 100644 --- a/src/lexer_v2/prop_tests.rs +++ b/src/lexer_v2/prop_tests.rs @@ -2,6 +2,8 @@ use super::{token::SpannedToken, StrTokenParser}; use proptest::{prelude::ProptestConfig, proptest}; +use alloc::string::String; + proptest! { #![proptest_config(ProptestConfig { cases: 30_000, diff --git a/src/lexer_v2/token/tests.rs b/src/lexer_v2/token/tests.rs index a3b4a3d..712221b 100644 --- a/src/lexer_v2/token/tests.rs +++ b/src/lexer_v2/token/tests.rs @@ -2,6 +2,8 @@ use super::super::StrTokenParser; use super::Token; use crate::{number, word}; +use alloc::vec::Vec; + macro_rules! test_tokens { ($(#[$($attr:tt)+])* $name:ident, ($($input:tt)*), [$($expected:expr),*]) => { ::paste::paste! { diff --git a/src/lexer_v2/word/mod.rs b/src/lexer_v2/word/mod.rs index 2f4300b..578e0fa 100644 --- a/src/lexer_v2/word/mod.rs +++ b/src/lexer_v2/word/mod.rs @@ -1,4 +1,4 @@ -use std::ops::Not; +use core::ops::Not; #[derive( Debug, diff --git a/src/lexer_v2/word/prop_tests.rs b/src/lexer_v2/word/prop_tests.rs index 8fecec5..3cde72f 100644 --- a/src/lexer_v2/word/prop_tests.rs +++ b/src/lexer_v2/word/prop_tests.rs @@ -2,6 +2,8 @@ use super::Word; use proptest::{prelude::ProptestConfig, prop_assert_eq, proptest}; +use alloc::string::String; + proptest! { #![proptest_config(ProptestConfig { cases: 10_000, diff --git a/src/lib.rs b/src/lib.rs index ebeb518..78631d5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,5 @@ #![doc = include_str!("../README.md")] +#![no_std] use core::fmt::Display; @@ -11,6 +12,10 @@ pub mod lexer_v2; #[macro_use] extern crate tracing; +// #[cfg(test)] // <-- TODO: make optional +#[macro_use] +extern crate alloc; + #[cfg(test)] #[allow(unused)] pub(crate) fn log_init() { @@ -47,4 +52,5 @@ pub(crate) fn display_opt(t: &Option) -> &dyn Display { } } #[cfg(test)] -extern crate alloc; +#[macro_use] +extern crate std; diff --git a/src/parser.rs b/src/parser.rs index 015554c..d9d7769 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -1,6 +1,11 @@ //! This module defines parser and relevant structs. -use std::num::NonZeroUsize; +use core::num::NonZeroUsize; + +use alloc::{ + string::{String, ToString}, + vec::Vec, +}; use nom::{ branch::alt, @@ -9,6 +14,7 @@ use nom::{ sequence::tuple, Err, Needed, }; + use num::complex::Complex64; use thiserror::Error; @@ -407,6 +413,7 @@ fn paren(tokens: &[Token]) -> Res<&[Token]> { #[cfg(test)] mod tests { + use alloc::borrow::ToOwned; use num::complex::Complex64; use rand::Rng; From 02147c4aee9e55a852ebf160de69f9e74e8e89d4 Mon Sep 17 00:00:00 2001 From: Dzuchun Date: Mon, 13 Jan 2025 03:59:26 +0200 Subject: [PATCH 5/6] Old-New tokens intertop --- src/evaluation_tree/mod.rs | 4 +- src/lexer.rs | 68 +++++----- src/parser.rs | 258 ++++++++++++++++++------------------- 3 files changed, 162 insertions(+), 168 deletions(-) diff --git a/src/evaluation_tree/mod.rs b/src/evaluation_tree/mod.rs index 0f334c7..edf5518 100644 --- a/src/evaluation_tree/mod.rs +++ b/src/evaluation_tree/mod.rs @@ -10,7 +10,7 @@ use alloc::{boxed::Box, string::String}; use num::{complex::Complex64, Complex}; use crate::{ - lexer::Token, + lexer::OldToken, parser::{parse, GenerationError}, }; @@ -49,7 +49,7 @@ pub struct EvaluationTree(Node); impl EvaluationTree { /// Attempts to parse a tree from token slice. May fail with [`GenerationError`] to indicate incorrect syntax. - pub fn from_tokens(tokens: &[Token]) -> Result { + pub fn from_tokens(tokens: &[OldToken]) -> Result { Ok(Self(parse(tokens)?)) } } diff --git a/src/lexer.rs b/src/lexer.rs index 8f02d10..9a10e45 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -24,7 +24,7 @@ use crate::{ }, }; -use core::{fmt::Display, str::FromStr}; +use core::str::FromStr; type Res<'l, T = OldToken> = nom::IResult<&'l str, T, Error<&'l str>>; @@ -51,7 +51,7 @@ pub enum Operator { /// Represents a single token. #[derive(Debug, PartialEq)] -pub enum Token { +pub enum OldToken { GroupOpen(GroupingType), GroupClose(GroupingType), Operator(Operator), @@ -61,8 +61,6 @@ pub enum Token { Identifier(String), } -<<<<<<< HEAD -======= #[derive(Debug, Clone, Copy, thiserror::Error)] #[error("Unsupported token: {_0}")] pub struct UnsupportedToken<'s>(Token<&'s str, &'s str, &'s str>); @@ -132,7 +130,6 @@ impl<'s> TryFrom> for OldToken { } } ->>>>>>> 7d0e9b0 (impl app) /// Represents token type (used for debug) #[derive(Debug, PartialEq, derive_more::Display)] pub enum TokenType { @@ -145,7 +142,7 @@ pub enum TokenType { fn paren_open(i: &str) -> Res { map(one_of("([{"), |c| { - Token::GroupOpen(match c { + OldToken::GroupOpen(match c { '(' => GroupingType::Parentheses, '[' => GroupingType::Brackets, '{' => GroupingType::Braces, @@ -156,7 +153,7 @@ fn paren_open(i: &str) -> Res { fn paren_close(i: &str) -> Res { map(one_of(")]}"), |c| { - Token::GroupClose(match c { + OldToken::GroupClose(match c { ')' => GroupingType::Parentheses, ']' => GroupingType::Brackets, '}' => GroupingType::Braces, @@ -167,7 +164,7 @@ fn paren_close(i: &str) -> Res { fn operator(i: &str) -> Res { map(one_of("+-*/^"), |c| { - Token::Operator(match c { + OldToken::Operator(match c { '+' => Operator::Plus, '-' => Operator::Minus, '*' => Operator::Star, @@ -179,11 +176,11 @@ fn operator(i: &str) -> Res { } fn comma(i: &str) -> Res { - map(pchar(','), |_| Token::Comma)(i) + map(pchar(','), |_| OldToken::Comma)(i) } fn imaginary_unit(i: &str) -> Res { - map(pchar('i'), |_| Token::ImaginaryUnit)(i) + map(pchar('i'), |_| OldToken::ImaginaryUnit)(i) } // source: https://github.com/rust-bakery/nom/blob/main/doc/nom_recipes.md#identifiers @@ -193,12 +190,12 @@ fn identifier(i: &str) -> Res { alt((alpha1, tag("_"))), many0_count(alt((alphanumeric1, tag("_")))), )), - |name: &str| Token::Identifier(name.to_owned()), + |name: &str| OldToken::Identifier(name.to_owned()), )(i) } fn number(i: &str) -> Res { - map(double, Token::Number)(i) + map(double, OldToken::Number)(i) } fn whitespace(i: &str) -> Res<()> { @@ -206,7 +203,7 @@ fn whitespace(i: &str) -> Res<()> { } /// lexes the input -pub fn lex(i: &str) -> Res> { +pub fn lex(i: &str) -> Res> { fold_many1( preceded( whitespace, @@ -230,18 +227,15 @@ pub fn lex(i: &str) -> Res> { #[cfg(test)] mod tests { -<<<<<<< HEAD -======= use alloc::borrow::ToOwned; use alloc::vec::Vec; ->>>>>>> 7d0e9b0 (impl app) use crate::lexer::GroupingType; use crate::lexer_v2::token::Token; use crate::lexer_v2::StrTokenParser; use super::lex; - use super::Token; + use super::OldToken; #[test] fn tokens() { @@ -257,26 +251,26 @@ mod tests { Ok(( "", vec![ - Token::Number(2f64), - Token::Operator(crate::lexer::Operator::Plus), - Token::Identifier("x".to_owned()), - Token::Operator(crate::lexer::Operator::Cap), - Token::Number(2f64), - Token::Operator(crate::lexer::Operator::Minus), - Token::GroupOpen(GroupingType::Parentheses), - Token::Number(3f64), - Token::Identifier("x".to_owned()), - Token::Operator(crate::lexer::Operator::Cap), - Token::Number(2f64), - Token::Operator(crate::lexer::Operator::Minus), - Token::Number(7f64), - Token::Identifier("y".to_owned()), - Token::Operator(crate::lexer::Operator::Slash), - Token::Identifier("sin".to_owned()), - Token::GroupOpen(GroupingType::Brackets), - Token::Identifier("z".to_owned()), - Token::GroupClose(GroupingType::Brackets), - Token::GroupClose(GroupingType::Parentheses), + OldToken::Number(2f64), + OldToken::Operator(crate::lexer::Operator::Plus), + OldToken::Identifier("x".to_owned()), + OldToken::Operator(crate::lexer::Operator::Cap), + OldToken::Number(2f64), + OldToken::Operator(crate::lexer::Operator::Minus), + OldToken::GroupOpen(GroupingType::Parentheses), + OldToken::Number(3f64), + OldToken::Identifier("x".to_owned()), + OldToken::Operator(crate::lexer::Operator::Cap), + OldToken::Number(2f64), + OldToken::Operator(crate::lexer::Operator::Minus), + OldToken::Number(7f64), + OldToken::Identifier("y".to_owned()), + OldToken::Operator(crate::lexer::Operator::Slash), + OldToken::Identifier("sin".to_owned()), + OldToken::GroupOpen(GroupingType::Brackets), + OldToken::Identifier("z".to_owned()), + OldToken::GroupClose(GroupingType::Brackets), + OldToken::GroupClose(GroupingType::Parentheses), ] )) ); diff --git a/src/parser.rs b/src/parser.rs index d9d7769..2673591 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -24,7 +24,7 @@ use crate::{ functions::NamedFunction, Constant, Node, Variable, }, - lexer::{Operator, Token, TokenType}, + lexer::{OldToken, Operator, TokenType}, }; /// Represents error occurred during tree generation. @@ -32,13 +32,13 @@ use crate::{ pub enum GenerationError<'l> { /// Error emitted by `nom` #[error("{0:?}")] - NomError(NError<&'l [Token]>), + NomError(NError<&'l [OldToken]>), /// There are no more tokens, but some were expected #[error("No tokens")] Empty(Needed), /// Unexpected token type encountered #[error("Bad token: {} expected, but {:?} found", .0, .1)] - BadToken(TokenType, &'l Token), + BadToken(TokenType, &'l OldToken), /// There's an unpaired grouping #[error("Paren was left unclosed")] UnpairedGrouping, @@ -47,12 +47,12 @@ pub enum GenerationError<'l> { NoToken, } -impl<'l> ParseError<&'l [Token]> for GenerationError<'l> { - fn from_error_kind(input: &'l [Token], kind: nom::error::ErrorKind) -> Self { +impl<'l> ParseError<&'l [OldToken]> for GenerationError<'l> { + fn from_error_kind(input: &'l [OldToken], kind: nom::error::ErrorKind) -> Self { Self::NomError(NError::new(input, kind)) } - fn append(_input: &[Token], _kind: nom::error::ErrorKind, other: Self) -> Self { + fn append(_input: &[OldToken], _kind: nom::error::ErrorKind, other: Self) -> Self { // not really sure what this is about, // but it seems that `nom` itself implements this function in a similar fashion, // so I guess it's not much of a harm. @@ -61,10 +61,10 @@ impl<'l> ParseError<&'l [Token]> for GenerationError<'l> { } } -type Res<'l, T = Node> = Result<(&'l [Token], T), Err>>; +type Res<'l, T = Node> = Result<(&'l [OldToken], T), Err>>; /// parses tokens into a node. -pub fn parse(tokens: &[Token]) -> Result, GenerationError> { +pub fn parse(tokens: &[OldToken]) -> Result, GenerationError> { all_consuming(node)(tokens) .map(|(_, n)| n) .map_err(|err| match err { @@ -73,12 +73,12 @@ pub fn parse(tokens: &[Token]) -> Result, GenerationError> { }) } -fn node(tokens: &[Token]) -> Res { +fn node(tokens: &[OldToken]) -> Res { assert_parens(tokens)?; alt((all_consuming(map_parser(paren, raw_node)), raw_node))(tokens) } -fn raw_node(tokens: &[Token]) -> Res { +fn raw_node(tokens: &[OldToken]) -> Res { alt(( all_consuming(function_call), all_consuming(variable), @@ -87,7 +87,7 @@ fn raw_node(tokens: &[Token]) -> Res { ))(tokens) } -fn bin_operator(tokens: &[Token]) -> Res { +fn bin_operator(tokens: &[OldToken]) -> Res { let (_, (node1, op, node2)) = find_bin_operator(tokens)?; let (_, node1) = node(node1)?; let (_, node2) = node(node2)?; @@ -103,7 +103,7 @@ fn bin_operator(tokens: &[Token]) -> Res { )) } -fn find_bin_operator(tokens: &[Token]) -> Res<(&[Token], Operator, &[Token])> { +fn find_bin_operator(tokens: &[OldToken]) -> Res<(&[OldToken], Operator, &[OldToken])> { static PREDESCENCE: [Operator; 5] = [ Operator::Plus, Operator::Minus, @@ -118,9 +118,9 @@ fn find_bin_operator(tokens: &[Token]) -> Res<(&[Token], Operator, &[Token])> { .enumerate() .filter_map(|(i, token)| { match token { - Token::GroupOpen(_) => inner += 1, - Token::GroupClose(_) => inner -= 1, - Token::Operator(op) if op == operator => { + OldToken::GroupOpen(_) => inner += 1, + OldToken::GroupClose(_) => inner -= 1, + OldToken::Operator(op) if op == operator => { if inner == 0 { return Some(i); } @@ -141,7 +141,7 @@ fn find_bin_operator(tokens: &[Token]) -> Res<(&[Token], Operator, &[Token])> { Err(Err::Error(GenerationError::NoToken)) } -fn constant(tokens: &[Token]) -> Res { +fn constant(tokens: &[OldToken]) -> Res { let (rest, val) = alt(( map( tuple((opt(sign), real_part, sign, imaginary_part)), @@ -171,11 +171,11 @@ fn constant(tokens: &[Token]) -> Res { Ok((rest, Constant(val).into())) } -fn real_part(tokens: &[Token]) -> Res { +fn real_part(tokens: &[OldToken]) -> Res { number(tokens) } -fn imaginary_part(tokens: &[Token]) -> Res { +fn imaginary_part(tokens: &[OldToken]) -> Res { alt(( map(tuple((number, opt(star), imaginary_unit)), |(n, _, ())| n), map(tuple((imaginary_unit, star, number)), |((), (), n)| n), @@ -183,7 +183,7 @@ fn imaginary_part(tokens: &[Token]) -> Res { ))(tokens) } -fn function_call(tokens: &[Token]) -> Res { +fn function_call(tokens: &[OldToken]) -> Res { map( tuple(( opt(sign), @@ -208,7 +208,7 @@ fn function_call(tokens: &[Token]) -> Res { )(tokens) } -fn variable(tokens: &[Token]) -> Res { +fn variable(tokens: &[OldToken]) -> Res { map(tuple((opt(sign), identifier)), |(minus, name)| { if let Some(true) = minus { Negation::new(Variable::new(name)).into() @@ -220,14 +220,14 @@ fn variable(tokens: &[Token]) -> Res { // general tokens -fn assert_parens(tokens: &[Token]) -> Res<()> { +fn assert_parens(tokens: &[OldToken]) -> Res<()> { let mut stack = Vec::new(); for token in tokens { - if let Token::GroupOpen(t) = token { + if let OldToken::GroupOpen(t) = token { stack.push(t); } - if let Token::GroupClose(t) = token { + if let OldToken::GroupClose(t) = token { let Some(&e) = stack.last() else { return Err(Err::Failure(GenerationError::UnpairedGrouping)); }; @@ -244,13 +244,13 @@ fn assert_parens(tokens: &[Token]) -> Res<()> { } } -fn number(tokens: &[Token]) -> Res { +fn number(tokens: &[OldToken]) -> Res { let token = tokens .first() .ok_or(Err::Error(GenerationError::Empty(Needed::Size( NonZeroUsize::try_from(1).expect("1 is not zero"), ))))?; - let Token::Number(val) = token else { + let OldToken::Number(val) = token else { return Err(Err::Error(GenerationError::BadToken( TokenType::Constant, token, @@ -259,13 +259,13 @@ fn number(tokens: &[Token]) -> Res { Ok((&tokens[1..], *val)) } -fn plus(tokens: &[Token]) -> Res<()> { +fn plus(tokens: &[OldToken]) -> Res<()> { let t = tokens .first() .ok_or(Err::Error(GenerationError::Empty(Needed::Size( NonZeroUsize::try_from(1).expect("1 is not zero"), ))))?; - let Token::Operator(Operator::Plus) = t else { + let OldToken::Operator(Operator::Plus) = t else { return Err(Err::Error(GenerationError::BadToken( TokenType::Operator, t, @@ -274,13 +274,13 @@ fn plus(tokens: &[Token]) -> Res<()> { Ok((&tokens[1..], ())) } -fn minus(tokens: &[Token]) -> Res<()> { +fn minus(tokens: &[OldToken]) -> Res<()> { let t = tokens .first() .ok_or(Err::Error(GenerationError::Empty(Needed::Size( NonZeroUsize::try_from(1).expect("1 is not zero"), ))))?; - let Token::Operator(Operator::Minus) = t else { + let OldToken::Operator(Operator::Minus) = t else { return Err(Err::Error(GenerationError::BadToken( TokenType::Operator, t, @@ -289,13 +289,13 @@ fn minus(tokens: &[Token]) -> Res<()> { Ok((&tokens[1..], ())) } -fn star(tokens: &[Token]) -> Res<()> { +fn star(tokens: &[OldToken]) -> Res<()> { let t = tokens .first() .ok_or(Err::Error(GenerationError::Empty(Needed::Size( NonZeroUsize::try_from(1).expect("1 is not zero"), ))))?; - let Token::Operator(Operator::Star) = t else { + let OldToken::Operator(Operator::Star) = t else { return Err(Err::Error(GenerationError::BadToken( TokenType::Operator, t, @@ -304,13 +304,13 @@ fn star(tokens: &[Token]) -> Res<()> { Ok((&tokens[1..], ())) } -fn imaginary_unit(tokens: &[Token]) -> Res<()> { +fn imaginary_unit(tokens: &[OldToken]) -> Res<()> { let t = tokens .first() .ok_or(Err::Error(GenerationError::Empty(Needed::Size( NonZeroUsize::try_from(1).expect("1 is not zero"), ))))?; - let Token::ImaginaryUnit = t else { + let OldToken::ImaginaryUnit = t else { return Err(Err::Error(GenerationError::BadToken( TokenType::Operator, t, @@ -319,13 +319,13 @@ fn imaginary_unit(tokens: &[Token]) -> Res<()> { Ok((&tokens[1..], ())) } -fn identifier(tokens: &[Token]) -> Res { +fn identifier(tokens: &[OldToken]) -> Res { let t = tokens .first() .ok_or(Err::Error(GenerationError::Empty(Needed::Size( NonZeroUsize::try_from(1).expect("1 is not zero"), ))))?; - let Token::Identifier(name) = t else { + let OldToken::Identifier(name) = t else { return Err(Err::Error(GenerationError::BadToken( TokenType::Operator, t, @@ -334,21 +334,21 @@ fn identifier(tokens: &[Token]) -> Res { Ok((&tokens[1..], name.to_string())) } -fn comma_separated(tokens: &[Token]) -> Res<(Node, Node)> { +fn comma_separated(tokens: &[OldToken]) -> Res<(Node, Node)> { let (_, (arg1, arg2)) = find_comma(tokens)?; let (_, arg1) = node(arg1)?; let (_, arg2) = node(arg2)?; Ok((&[], (arg1, arg2))) } -fn find_comma(tokens: &[Token]) -> Res<(&[Token], &[Token])> { +fn find_comma(tokens: &[OldToken]) -> Res<(&[OldToken], &[OldToken])> { let mut inline = Vec::new(); for (i, token) in tokens.iter().enumerate() { match token { - Token::GroupOpen(t) => { + OldToken::GroupOpen(t) => { inline.push(t); } - Token::GroupClose(t) => { + OldToken::GroupClose(t) => { if &t != inline .last() @@ -359,7 +359,7 @@ fn find_comma(tokens: &[Token]) -> Res<(&[Token], &[Token])> { inline.pop(); } - Token::Comma => { + OldToken::Comma => { if inline.is_empty() { return Ok((&[], (&tokens[..i], &tokens[i + 1..]))); } @@ -370,17 +370,17 @@ fn find_comma(tokens: &[Token]) -> Res<(&[Token], &[Token])> { Err(Err::Error(GenerationError::NoToken)) } -fn sign(tokens: &[Token]) -> Res { +fn sign(tokens: &[OldToken]) -> Res { alt((map(plus, |()| false), map(minus, |()| true)))(tokens) } -fn paren(tokens: &[Token]) -> Res<&[Token]> { +fn paren(tokens: &[OldToken]) -> Res<&[OldToken]> { let t = tokens .first() .ok_or(Err::Error(GenerationError::Empty(Needed::Size( NonZeroUsize::try_from(1).expect("1 is not zero"), ))))?; - let Token::GroupOpen(t) = t else { + let OldToken::GroupOpen(t) = t else { return Err(Err::Error(GenerationError::BadToken( TokenType::Grouping, t, @@ -390,12 +390,12 @@ fn paren(tokens: &[Token]) -> Res<&[Token]> { let mut inner = 1; for i in 1..tokens.len() { let token = &tokens[i]; - if let Token::GroupOpen(e) = token { + if let OldToken::GroupOpen(e) = token { if t == e { inner += 1; } } - if let Token::GroupClose(e) = token { + if let OldToken::GroupClose(e) = token { if t == e { inner -= 1; @@ -424,7 +424,7 @@ mod tests { functions::NamedFunction, Constant, Evaluatable, Node, Variable, }, - lexer::{GroupingType, Operator, Token}, + lexer::{GroupingType, OldToken, Operator}, }; use super::parse; @@ -489,122 +489,122 @@ mod tests { // following tests reinforce that complex constant recognition works as expected: test! {constant1, - tokens = [Token::Number(1.0f64)], + tokens = [OldToken::Number(1.0f64)], expected = Constant::from(1.0f64)} test! {constant2, - tokens = [Token::ImaginaryUnit], + tokens = [OldToken::ImaginaryUnit], expected = Constant::from(1.0f64).tr()} test! {constant3, tokens = [ - Token::Number(1.0f64), - Token::Operator(Operator::Star), - Token::ImaginaryUnit + OldToken::Number(1.0f64), + OldToken::Operator(Operator::Star), + OldToken::ImaginaryUnit ], expected = Constant::from(1.0f64).tr()} test! {constant4, tokens = [ - Token::Number(1.0f64), - Token::Operator(Operator::Plus), - Token::ImaginaryUnit + OldToken::Number(1.0f64), + OldToken::Operator(Operator::Plus), + OldToken::ImaginaryUnit ], expected = Constant(Complex64::new(1.0, 1.0))} test! {constant5, tokens = [ - Token::Operator(Operator::Minus), - Token::Number(1.0f64), - Token::Operator(Operator::Plus), - Token::Number(1.0), - Token::Operator(Operator::Star), - Token::ImaginaryUnit + OldToken::Operator(Operator::Minus), + OldToken::Number(1.0f64), + OldToken::Operator(Operator::Plus), + OldToken::Number(1.0), + OldToken::Operator(Operator::Star), + OldToken::ImaginaryUnit ], expected = Constant(Complex64::new(-1.0, 1.0))} test! {constant6, tokens = [ - Token::Operator(Operator::Minus), - Token::Number(1.0f64), - Token::Operator(Operator::Plus), - Token::Number(1.0), Token::ImaginaryUnit + OldToken::Operator(Operator::Minus), + OldToken::Number(1.0f64), + OldToken::Operator(Operator::Plus), + OldToken::Number(1.0), OldToken::ImaginaryUnit ], expected = Constant(Complex64::new(-1.0, 1.0))} test! {constant7, tokens = [ - Token::Operator(Operator::Minus), - Token::ImaginaryUnit, - Token::Operator(Operator::Star), - Token::Number(1.0f64), - Token::Operator(Operator::Plus), - Token::Number(1.0) + OldToken::Operator(Operator::Minus), + OldToken::ImaginaryUnit, + OldToken::Operator(Operator::Star), + OldToken::Number(1.0f64), + OldToken::Operator(Operator::Plus), + OldToken::Number(1.0) ], expected = Constant(Complex64::new(1.0, -1.0))} test! {constant8, tokens = [ - Token::Operator(Operator::Minus), - Token::ImaginaryUnit, - Token::Operator(Operator::Plus), - Token::Number(1.0) + OldToken::Operator(Operator::Minus), + OldToken::ImaginaryUnit, + OldToken::Operator(Operator::Plus), + OldToken::Number(1.0) ], expected = Constant(Complex64::new(1.0, -1.0))} test! {constant9, tokens = [ - Token::Operator(Operator::Minus), - Token::ImaginaryUnit + OldToken::Operator(Operator::Minus), + OldToken::ImaginaryUnit ], expected = Constant(Complex64::new(0.0, -1.0))} // following tests reinforce that variable recognition works as expected: test! {variable1, - tokens = [Token::Identifier("x".to_owned())], + tokens = [OldToken::Identifier("x".to_owned())], expected = Variable::new("x")} test! {variable2, - tokens = [Token::Identifier("a_123".to_owned())], + tokens = [OldToken::Identifier("a_123".to_owned())], expected = Variable::new("a_123")} // following tests reinforce that function calls work as expected: test! {function1, tokens = [ - Token::Identifier("sin".to_owned()), - Token::GroupOpen(GroupingType::Parentheses), - Token::Identifier("x".to_owned()), - Token::GroupClose(GroupingType::Parentheses) + OldToken::Identifier("sin".to_owned()), + OldToken::GroupOpen(GroupingType::Parentheses), + OldToken::Identifier("x".to_owned()), + OldToken::GroupClose(GroupingType::Parentheses) ], expected = NamedFunction::new("sin", Variable::new("x"))} test! {function2, tokens = [ - Token::Identifier("sum_sq".to_owned()), - Token::GroupOpen(GroupingType::Parentheses), - Token::Identifier("x".to_owned()), - Token::Comma, - Token::Identifier("y".to_owned()), - Token::GroupClose(GroupingType::Parentheses) + OldToken::Identifier("sum_sq".to_owned()), + OldToken::GroupOpen(GroupingType::Parentheses), + OldToken::Identifier("x".to_owned()), + OldToken::Comma, + OldToken::Identifier("y".to_owned()), + OldToken::GroupClose(GroupingType::Parentheses) ], expected = NamedFunction::new("sum_sq", (Variable::new("x"), Variable::new("y")))} test! {function3, tokens = [ - Token::Operator(Operator::Minus), - Token::Identifier("sin".to_owned()), - Token::GroupOpen(GroupingType::Parentheses), - Token::Identifier("x".to_owned()), - Token::GroupClose(GroupingType::Parentheses) + OldToken::Operator(Operator::Minus), + OldToken::Identifier("sin".to_owned()), + OldToken::GroupOpen(GroupingType::Parentheses), + OldToken::Identifier("x".to_owned()), + OldToken::GroupClose(GroupingType::Parentheses) ], expected = Negation::new(NamedFunction::new("sin", Variable::new("x")))} // following tests reinforce correct binary operator parsing // x + y * 2 test! {bin_operator1, tokens = [ - Token::Identifier("x".to_owned()), - Token::Operator(Operator::Plus), - Token::Identifier("y".to_owned()), - Token::Operator(Operator::Star), - Token::Number(2f64) + OldToken::Identifier("x".to_owned()), + OldToken::Operator(Operator::Plus), + OldToken::Identifier("y".to_owned()), + OldToken::Operator(Operator::Star), + OldToken::Number(2f64) ], expected = Addition::new(Variable::new("x"), Multiplication::new(Variable::new("y"), Constant(2f64.into())))} // [-x + 4] * sin{y}^2 test! {bin_operator2, tokens = [ - Token::GroupOpen(GroupingType::Brackets), - Token::Operator(Operator::Minus), - Token::Identifier("x".to_owned()), - Token::Operator(Operator::Plus), - Token::Number(4f64), - Token::GroupClose(GroupingType::Brackets), - Token::Operator(Operator::Star), - Token::Identifier("sin".to_owned()), - Token::GroupOpen(GroupingType::Braces), - Token::Identifier("y".to_owned()), - Token::GroupClose(GroupingType::Braces), - Token::Operator(Operator::Cap), - Token::Number(2f64) + OldToken::GroupOpen(GroupingType::Brackets), + OldToken::Operator(Operator::Minus), + OldToken::Identifier("x".to_owned()), + OldToken::Operator(Operator::Plus), + OldToken::Number(4f64), + OldToken::GroupClose(GroupingType::Brackets), + OldToken::Operator(Operator::Star), + OldToken::Identifier("sin".to_owned()), + OldToken::GroupOpen(GroupingType::Braces), + OldToken::Identifier("y".to_owned()), + OldToken::GroupClose(GroupingType::Braces), + OldToken::Operator(Operator::Cap), + OldToken::Number(2f64) ], expected = Multiplication::new( Addition::new( Negation::new(Variable::new("x")), @@ -621,24 +621,24 @@ mod tests { // z ^ [z] + sin(1.0 / e ^ {i * phi}) test! {bin_operator3, tokens = [ - Token::Identifier("z".to_owned()), - Token::Operator(Operator::Cap), - Token::GroupOpen(GroupingType::Brackets), - Token::Identifier("z".to_owned()), - Token::GroupClose(GroupingType::Brackets), - Token::Operator(Operator::Plus), - Token::Identifier("sin".to_owned()), - Token::GroupOpen(GroupingType::Parentheses), - Token::Number(1.0), - Token::Operator(Operator::Slash), - Token::Identifier("e".to_owned()), - Token::Operator(Operator::Cap), - Token::GroupOpen(GroupingType::Braces), - Token::ImaginaryUnit, - Token::Operator(Operator::Star), - Token::Identifier("phi".to_owned()), - Token::GroupClose(GroupingType::Braces), - Token::GroupClose(GroupingType::Parentheses) + OldToken::Identifier("z".to_owned()), + OldToken::Operator(Operator::Cap), + OldToken::GroupOpen(GroupingType::Brackets), + OldToken::Identifier("z".to_owned()), + OldToken::GroupClose(GroupingType::Brackets), + OldToken::Operator(Operator::Plus), + OldToken::Identifier("sin".to_owned()), + OldToken::GroupOpen(GroupingType::Parentheses), + OldToken::Number(1.0), + OldToken::Operator(Operator::Slash), + OldToken::Identifier("e".to_owned()), + OldToken::Operator(Operator::Cap), + OldToken::GroupOpen(GroupingType::Braces), + OldToken::ImaginaryUnit, + OldToken::Operator(Operator::Star), + OldToken::Identifier("phi".to_owned()), + OldToken::GroupClose(GroupingType::Braces), + OldToken::GroupClose(GroupingType::Parentheses) ], expected = Addition::new( Exponentiation::new( Variable::new("z"), From 243b6ad0035c2969a76ab8163ced5dc0fbb04ea1 Mon Sep 17 00:00:00 2001 From: Dzuchun Date: Mon, 13 Jan 2025 04:53:23 +0200 Subject: [PATCH 6/6] Version bump --- Cargo.lock | 2 +- Cargo.toml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f09dc27..a44a3dd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -168,7 +168,7 @@ checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" [[package]] name = "mathvis" -version = "0.3.0" +version = "0.4.0" dependencies = [ "derive_more", "generic-array", diff --git a/Cargo.toml b/Cargo.toml index b790378..c5dfa15 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mathvis" -version = "0.3.0" +version = "0.4.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -10,7 +10,7 @@ derive_more = {version = "1.0.0", features = ["display", "as_ref", "deref"]} nom = "7.1.3" num = "0.4.3" thiserror = "2.0.11" -generic-array = "1.1.1" +generic-array = "1.2.0" tracing = { version = "0.1.41", default-features = false, features = ["attributes"] } hashbrown = "0.15.2"