From a4e75196c6a5fb554e79f8bed34c20d96256cb09 Mon Sep 17 00:00:00 2001 From: Dzuchun Date: Mon, 13 Jan 2025 03:53:31 +0200 Subject: [PATCH 1/8] Ignored proptest files --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index ea8c4bf..60f7a9c 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ /target +/proptest-regressions From 4bde45ef698e88d19e4b147ba1c4cdc737f3b6a0 Mon Sep 17 00:00:00 2001 From: Dzuchun Date: Mon, 13 Jan 2025 04:07:58 +0200 Subject: [PATCH 2/8] [Cargo.lock] --- Cargo.lock | 485 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 483 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b8c7db0..f09dc27 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,12 +2,39 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "autocfg" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + +[[package]] +name = "bitflags" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1be3f42a67d6d345ecd59f675f3f012d6974981560836e938c22b424b85ce1be" + [[package]] name = "byteorder" version = "1.5.0" @@ -20,6 +47,15 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", +] + [[package]] name = "derive_more" version = "1.0.0" @@ -41,6 +77,49 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "errno" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0d2fde1f7b3d48b8395d5f2de76c18a528bd6a9cdde438df747bfcba3e05d6f" + +[[package]] +name = "generic-array" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8c8444bc9d71b935156cc0ccab7f622180808af7867b1daae6547d773591703" +dependencies = [ + "typenum", +] + [[package]] name = "getrandom" version = "0.2.15" @@ -52,21 +131,56 @@ dependencies = [ "wasi", ] +[[package]] +name = "hashbrown" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "itoa" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + [[package]] name = "libc" version = "0.2.169" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + [[package]] name = "mathvis" version = "0.3.0" dependencies = [ "derive_more", + "generic-array", + "hashbrown", "nom", "num", + "paste", + "proptest", "rand", "thiserror", + "tracing", + "tracing-subscriber", ] [[package]] @@ -91,6 +205,16 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + [[package]] name = "num" version = "0.4.3" @@ -124,6 +248,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-integer" version = "0.1.46" @@ -164,6 +294,45 @@ dependencies = [ "autocfg", ] +[[package]] +name = "num_threads" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" +dependencies = [ + "libc", +] + +[[package]] +name = "once_cell" +version = "1.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" + +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.20" @@ -175,13 +344,39 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.92" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" +checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99" dependencies = [ "unicode-ident", ] +[[package]] +name = "proptest" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14cae93065090804185d3b75f0bf93b8eeda30c7a9b4a33d3bdb3988d6229e50" +dependencies = [ + "bit-set", + "bit-vec", + "bitflags", + "lazy_static", + "num-traits", + "rand", + "rand_chacha", + "rand_xorshift", + "regex-syntax", + "rusty-fork", + "tempfile", + "unarray", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + [[package]] name = "quote" version = "1.0.38" @@ -221,6 +416,75 @@ dependencies = [ "getrandom", ] +[[package]] +name = "rand_xorshift" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f" +dependencies = [ + "rand_core", +] + +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + +[[package]] +name = "rustix" +version = "0.38.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a78891ee6bf2340288408954ac787aa063d8e8817e9f53abb37c695c6d834ef6" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rusty-fork" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb3dcc6e454c328bb824492db107ab7c0ae8fcffe4ad210136ef014458c1bc4f" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] + +[[package]] +name = "serde" +version = "1.0.217" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.217" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + [[package]] name = "syn" version = "2.0.96" @@ -232,6 +496,20 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tempfile" +version = "3.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8a559c81686f576e8cd0290cd2a24a2a9ad80c98b3478856500fcbd7acd704" +dependencies = [ + "cfg-if", + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + [[package]] name = "thiserror" version = "2.0.11" @@ -252,6 +530,105 @@ dependencies = [ "syn", ] +[[package]] +name = "thread_local" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" +dependencies = [ + "cfg-if", + "once_cell", +] + +[[package]] +name = "time" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21" +dependencies = [ + "deranged", + "itoa", + "libc", + "num-conv", + "num_threads", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2834e6017e3e5e4b9834939793b282bc03b37a3336245fa820e35e233e2a85de" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tracing" +version = "0.1.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" +dependencies = [ + "once_cell", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" +dependencies = [ + "nu-ansi-term", + "sharded-slab", + "thread_local", + "time", + "tracing-core", +] + +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + [[package]] name = "unicode-ident" version = "1.0.14" @@ -264,12 +641,116 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "wait-timeout" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6" +dependencies = [ + "libc", +] + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + [[package]] name = "zerocopy" version = "0.7.35" From 376dd7ce1a7ed24c86aac6d7278e8bde590dfef9 Mon Sep 17 00:00:00 2001 From: Dzuchun Date: Mon, 13 Jan 2025 03:11:37 +0200 Subject: [PATCH 3/8] Implemented new lexing --- Cargo.lock | 30 -- Cargo.toml | 14 +- src/lexer.rs | 112 ++++++++ src/lexer_v2/mod.rs | 68 +++++ src/lexer_v2/number/mod.rs | 222 ++++++++++++++ src/lexer_v2/number/prop_tests.rs | 119 ++++++++ src/lexer_v2/number/tests.rs | 335 +++++++++++++++++++++ src/lexer_v2/prop_tests.rs | 17 ++ src/lexer_v2/token/mod.rs | 463 ++++++++++++++++++++++++++++++ src/lexer_v2/token/tests.rs | 164 +++++++++++ src/lexer_v2/word/mod.rs | 70 +++++ src/lexer_v2/word/prop_tests.rs | 35 +++ src/lexer_v2/word/tests.rs | 36 +++ src/lib.rs | 44 +++ 14 files changed, 1698 insertions(+), 31 deletions(-) create mode 100644 src/lexer_v2/mod.rs create mode 100644 src/lexer_v2/number/mod.rs create mode 100644 src/lexer_v2/number/prop_tests.rs create mode 100644 src/lexer_v2/number/tests.rs create mode 100644 src/lexer_v2/prop_tests.rs create mode 100644 src/lexer_v2/token/mod.rs create mode 100644 src/lexer_v2/token/tests.rs create mode 100644 src/lexer_v2/word/mod.rs create mode 100644 src/lexer_v2/word/prop_tests.rs create mode 100644 src/lexer_v2/word/tests.rs diff --git a/Cargo.lock b/Cargo.lock index f09dc27..a9223be 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,12 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "allocator-api2" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" - [[package]] name = "autocfg" version = "1.4.0" @@ -77,12 +71,6 @@ dependencies = [ "unicode-xid", ] -[[package]] -name = "equivalent" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" - [[package]] name = "errno" version = "0.3.10" @@ -105,12 +93,6 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "foldhash" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0d2fde1f7b3d48b8395d5f2de76c18a528bd6a9cdde438df747bfcba3e05d6f" - [[package]] name = "generic-array" version = "1.2.0" @@ -131,17 +113,6 @@ dependencies = [ "wasi", ] -[[package]] -name = "hashbrown" -version = "0.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" -dependencies = [ - "allocator-api2", - "equivalent", - "foldhash", -] - [[package]] name = "itoa" version = "1.0.14" @@ -172,7 +143,6 @@ version = "0.3.0" dependencies = [ "derive_more", "generic-array", - "hashbrown", "nom", "num", "paste", diff --git a/Cargo.toml b/Cargo.toml index 443f6b1..2247cea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,10 +6,22 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -derive_more = {version = "1.0.0", features = ["display"]} +derive_more = {version = "1.0.0", features = ["display", "as_ref", "deref"]} nom = "7.1.3" num = "0.4.3" thiserror = "2.0.11" +generic-array = "1.1.1" +tracing = { version = "0.1.41", default-features = false, features = ["attributes"] } [dev-dependencies] +paste = "1.0.15" rand = "0.8.5" +tracing-subscriber = { version = "0.3.19", default-features = false, features = ["std", "fmt", "time", "local-time", "ansi"] } +proptest = "1.6.0" + +[features] +default = ["std"] +std = [] + +[lints.rust] +unexpected_cfgs = { level = "warn", check-cfg = ['cfg(verbose_logging)'] } diff --git a/src/lexer.rs b/src/lexer.rs index 9221718..41dcc7f 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -11,6 +11,16 @@ use nom::{ sequence::{pair, preceded}, }; +use core::str::FromStr; + +use crate::{ + display_opt, + lexer_v2::{ + number::Number, + token::{SpecialKind, Token}, + }, +}; + type Res<'l, T = Token> = nom::IResult<&'l str, T, Error<&'l str>>; /// Represents type of grouping. @@ -46,6 +56,78 @@ pub enum Token { Identifier(String), } +<<<<<<< HEAD +======= +#[derive(Debug, Clone, Copy, thiserror::Error)] +#[error("Unsupported token: {_0}")] +pub struct UnsupportedToken<'s>(Token<&'s str, &'s str, &'s str>); + +impl<'s> TryFrom> for OldToken { + type Error = UnsupportedToken<'s>; + + fn try_from(value: Token<&'s str, &'s str, &'s str>) -> Result { + let token = match value { + Token::Grouping { + kind: crate::lexer_v2::token::GroupingKind::Parentheses, + side: crate::lexer_v2::token::GroupingSide::Left, + } => Self::GroupOpen(GroupingType::Parentheses), + Token::Grouping { + kind: crate::lexer_v2::token::GroupingKind::Brackets, + side: crate::lexer_v2::token::GroupingSide::Left, + } => Self::GroupOpen(GroupingType::Brackets), + Token::Grouping { + kind: crate::lexer_v2::token::GroupingKind::Braces, + side: crate::lexer_v2::token::GroupingSide::Left, + } => Self::GroupOpen(GroupingType::Braces), + Token::Grouping { + kind: crate::lexer_v2::token::GroupingKind::Parentheses, + side: crate::lexer_v2::token::GroupingSide::Right, + } => Self::GroupClose(GroupingType::Parentheses), + Token::Grouping { + kind: crate::lexer_v2::token::GroupingKind::Brackets, + side: crate::lexer_v2::token::GroupingSide::Right, + } => Self::GroupClose(GroupingType::Brackets), + Token::Grouping { + kind: crate::lexer_v2::token::GroupingKind::Braces, + side: crate::lexer_v2::token::GroupingSide::Right, + } => Self::GroupClose(GroupingType::Braces), + Token::Special(SpecialKind::Comma) => Self::Comma, + Token::Special(SpecialKind::Plus) => Self::Operator(Operator::Plus), + Token::Special(SpecialKind::Minus) => Self::Operator(Operator::Minus), + Token::Special(SpecialKind::Star) => Self::Operator(Operator::Star), + Token::Special(SpecialKind::Slash) => Self::Operator(Operator::Slash), + Token::Special(SpecialKind::Cap) => Self::Operator(Operator::Cap), + Token::Number(Number::Integer { int, exp }) => Self::Number( + f64::from_str(&format!("{int}.0{}", display_opt(&exp))) + .map_err(|_| UnsupportedToken(value))?, + ), + Token::Number(Number::Decimal { + int, + point, + frac, + exp, + }) => Self::Number( + f64::from_str(&format!("{int}{point}{frac}{}", display_opt(&exp))) + .map_err(|_| UnsupportedToken(value))?, + ), + Token::Number(Number::DecimalInt { int, point, exp }) => Self::Number( + f64::from_str(&format!("{int}{point}0{}", display_opt(&exp))) + .map_err(|_| UnsupportedToken(value))?, + ), + Token::Number(Number::DecimalFraction { point, frac, exp }) => Self::Number( + f64::from_str(&format!("0{point}{frac}{}", display_opt(&exp))) + .map_err(|_| UnsupportedToken(value))?, + ), + Token::Word(word) if *word == "i" => Self::ImaginaryUnit, + Token::Word(word) => Self::Identifier(word.to_string()), + _ => return Err(UnsupportedToken(value)), + }; + + Ok(token) + } +} + +>>>>>>> 7d0e9b0 (impl app) /// Represents token type (used for debug) #[derive(Debug, PartialEq, derive_more::Display)] pub enum TokenType { @@ -143,7 +225,15 @@ pub fn lex(i: &str) -> Res> { #[cfg(test)] mod tests { +<<<<<<< HEAD +======= + use alloc::borrow::ToOwned; + use alloc::vec::Vec; + +>>>>>>> 7d0e9b0 (impl app) use crate::lexer::GroupingType; + use crate::lexer_v2::token::Token; + use crate::lexer_v2::StrTokenParser; use super::lex; use super::Token; @@ -186,4 +276,26 @@ mod tests { )) ); } + + #[test] + fn tokens_new_to_old_same() { + crate::log_init(); + + // arrange + let input = "2 + x^2 - (3x^2 - 7y / sin[z])"; + + // act + let tokens = lex(input).expect("Must be a valid input").1; + let new_tokens = StrTokenParser::::new(input) + .inspect(|new_token| trace!(?new_token)) + .map(|p| { + p.expect("Should not encounter errors") + .try_into() + .expect("Should not have unsupported tokens") + }) + .collect::>(); + + // assert + assert_eq!(tokens, new_tokens); + } } diff --git a/src/lexer_v2/mod.rs b/src/lexer_v2/mod.rs new file mode 100644 index 0000000..2dbb9e9 --- /dev/null +++ b/src/lexer_v2/mod.rs @@ -0,0 +1,68 @@ +use core::marker::PhantomData; + +pub mod number; +pub mod token; +pub mod word; + +pub trait TokenInput { + fn is_empty(&self) -> bool; + fn on_invalid(self) -> Self; +} + +pub trait Parseable<'s>: Sized + 's { + fn try_parse(input: &'s str) -> Option<(Self, &'s str)>; +} + +impl<'s, T> StrTokenParser<'s, T> { + pub fn new(input: &'s str) -> Self { + Self { + input, + _phantom: PhantomData, + } + } +} + +pub struct StrTokenParser<'s, T> { + input: &'s str, + _phantom: PhantomData, +} + +fn skip_err(s: &str) -> (&str, &str) { + let next_whitespace = s + .char_indices() + .find_map(|(pos, ch)| ch.is_whitespace().then_some(pos)) + .unwrap_or(s.len()); + s.split_at(next_whitespace) +} + +impl<'s, T: Parseable<'s>> Iterator for StrTokenParser<'s, T> { + type Item = Result; + + #[tracing::instrument(level = "trace", skip(self), name = "token_parser", fields(input = self.input))] + fn next(&mut self) -> Option { + let ws = self.input.trim_start(); + trace!(ws); + if ws.is_empty() { + // no more tokens, exhausted the input + return None; + } + + if let Some((token, rest)) = T::try_parse(ws) { + // successful token parse, return the token + trace!("Parsed token"); + self.input = rest; + Some(Ok(token)) + } else { + // failed to parse the token + // + // skip to next whitespace character + let (erroneous, new_input) = skip_err(ws); + self.input = new_input; + trace!(erroneous); + Some(Err(erroneous)) + } + } +} + +#[cfg(test)] +mod prop_tests; diff --git a/src/lexer_v2/number/mod.rs b/src/lexer_v2/number/mod.rs new file mode 100644 index 0000000..c4d0a4a --- /dev/null +++ b/src/lexer_v2/number/mod.rs @@ -0,0 +1,222 @@ +use core::fmt::Display; +use std::ops::Not; + +use crate::display_opt; + +#[inline] +fn valid_sign(ch: char) -> bool { + matches!(ch, '+' | '-') +} + +#[inline] +fn valid_exponent_marker(ch: char) -> bool { + matches!(ch, 'e' | 'E') +} + +#[inline] +fn valid_decimal_point(ch: char) -> bool { + matches!(ch, '.') +} + +#[derive( + Debug, + Clone, + Copy, + Hash, + PartialEq, + Eq, + derive_more::AsRef, + derive_more::Deref, + derive_more::Display, +)] +#[display("{_0}")] +pub struct Digits(D); + +impl<'s> Digits<&'s str> { + #[tracing::instrument(level = "trace", ret, skip(input), fields(digits_input = input))] + fn parse(input: &'s str) -> Option<(Self, &'s str)> { + let end_pos = input + .char_indices() + .find_map(|(pos, ch)| ch.is_ascii_digit().not().then_some(pos)) + .unwrap_or(input.len()); + + if end_pos == 0 { + None + } else { + Some((Self(&input[..end_pos]), &input[end_pos..])) + } + } +} + +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, derive_more::Display)] +#[display("{e}{}{digits}", if let Some(s) = sign.as_ref() { s as &dyn Display } else { &"" as &dyn Display })] +pub struct Exponent { + e: C, + sign: Option, + digits: Digits, +} + +impl<'s> Exponent<&'s str, &'s str> { + #[tracing::instrument(level = "trace", ret, skip(input), fields(exponent_input = input))] + fn parse(input: &'s str) -> Option<(Self, &'s str)> { + let mut char_inds = input.char_indices(); + // if there's no first char, there's no exponent + let (_, e_marker) = char_inds.next()?; + if !valid_exponent_marker(e_marker) { + // does not start with an exponent marker, bail + return None; + } + + // exponent marker cannot go by itself + let (mut pos, maybe_sign) = char_inds.next()?; + let e = &input[..pos]; + // figure out sign + let sign; + if valid_sign(maybe_sign) { + // second char is a valid sign, query for next one + let (sign_end, _) = char_inds.next()?; + sign = Some(&input[pos..sign_end]); + pos = sign_end; + } else { + sign = None; + } + let (digits, rest) = Digits::parse(&input[pos..])?; + Some((Self { e, sign, digits }, rest)) + } +} + +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, derive_more::Display)] +pub enum Number { + #[display("int({int}e{})", display_opt(exp))] + Integer { + int: Digits, + exp: Option>, + }, + #[display("float({int}{point}{frac}{})", display_opt(exp))] + Decimal { + int: Digits, + point: C, + frac: Digits, + exp: Option>, + }, + #[display("float({int}{point}{})", display_opt(exp))] + DecimalInt { + int: Digits, + point: C, + // no frac part + exp: Option>, + }, + #[display("float({point}{})", display_opt(exp))] + DecimalFraction { + // no int part + point: C, + frac: Digits, + exp: Option>, + }, +} + +impl<'s> Number<&'s str, &'s str> { + #[tracing::instrument(level = "trace", skip(input), fields(number_input = input), ret)] + pub fn parse(mut input: &'s str) -> Option<(Self, &'s str)> { + let int_part: Option>; + + let frac_part: Option>; + let exp: Option>; + + if let Some((digits, rest)) = Digits::parse(input) { + int_part = Some(digits); + input = rest; + trace!(?int_part, input); + } else { + int_part = None; + } + + let point = 'point: { + // try parsing decimal point. if it's only a (optional) sign and integer part - it's a normal integer + let mut char_inds = input.char_indices(); + let Some((_, maybe_point)) = char_inds.next() else { + break 'point None; + }; + trace!(%maybe_point); + + // try parsing the sign + if !valid_decimal_point(maybe_point) { + break 'point None; + } + + // this is a valid point char, find there it ends + let point; + if let Some((point_end, _)) = char_inds.next() { + point = &input[..point_end]; + input = &input[point_end..]; + } else { + point = input; + input = ""; + }; + Some(point) + }; + trace!(?point, input); + + // try parsing fractional part + if let Some((digits, rest)) = Digits::parse(input) { + frac_part = Some(digits); + input = rest; + trace!(?frac_part, input); + } else { + frac_part = None; + } + + // try parsing exponent + if let Some((exponent, rest)) = Exponent::parse(input) { + exp = Some(exponent); + input = rest; + trace!(?exp, input); + } else { + exp = None; + } + + trace!(?int_part, ?point, ?frac_part); + match (int_part, point, frac_part) { + (Some(int), None, None) => Some((Self::Integer { int, exp }, input)), + (Some(int), Some(point), Some(frac)) => Some(( + Self::Decimal { + int, + point, + frac, + exp, + }, + input, + )), + (None, Some(point), Some(frac)) => { + Some((Self::DecimalFraction { point, frac, exp }, input)) + } + (Some(int), Some(point), None) => Some((Self::DecimalInt { int, point, exp }, input)), + _ => None, + } + } +} + +#[macro_export] +macro_rules! number { + (? $input:literal) => { + 'number: { + let Some((number, rest)) = $crate::lexer_v2::number::Number::parse(stringify!($input)) + else { + break 'number None; + }; + if !rest.trim().is_empty() { + break 'number None; + } + Some(number) + } + }; + ($input:literal) => { + number!(?$input).expect("Invalid number") + }; +} + +#[cfg(test)] +mod tests; + +#[cfg(test)] +mod prop_tests; diff --git a/src/lexer_v2/number/prop_tests.rs b/src/lexer_v2/number/prop_tests.rs new file mode 100644 index 0000000..54444ed --- /dev/null +++ b/src/lexer_v2/number/prop_tests.rs @@ -0,0 +1,119 @@ +use core::ops::Not; + +use super::{Digits, Exponent, Number}; + +use proptest::{prelude::ProptestConfig, prop_assert_eq, proptest}; + +proptest! { + #![proptest_config(ProptestConfig { + cases: 10_000, + max_default_size_range: 200, + ..ProptestConfig::default() + })] + #[test] + fn digits_never_panics(input: String) { + let _ = Digits::parse(&input); + } + + #[test] + fn digits_never_parse_letters(s in "[^0-9]+") { + prop_assert_eq!(Digits::parse(&s), None); + } + + #[test] + fn digits_always_parse_digits(digits in "[0-9]+", garbage in "[^0-9]+") { + let input = format!("{digits}{garbage}"); + prop_assert_eq!(Digits::parse(&input), Some((Digits(digits.as_str()), garbage.as_str()))); + } + + #[test] + fn exponent_never_panics(input: String) { + let _ = Exponent::parse(&input); + } + + #[test] + fn exponent_valid_parses(e in "[eE]", sign in "[+-]?", digits in "[0-9]+", sep in "[^0-9]", garbage: String) { + let input = format!("{e}{sign}{digits}{sep}{garbage}"); + let garbage = format!("{sep}{garbage}"); + prop_assert_eq!( + Exponent::parse(&input), + Some(( + Exponent{ + e: e.as_str(), + sign: sign.is_empty().not().then_some(sign.as_str()), + digits: Digits(digits.as_str()) + }, + garbage.as_str()) + )); + } + + #[test] + fn number_never_panics(input: String) { + let _ = Number::parse(&input); + } + + #[test] + fn number_parses_ints(digits in "[0-9]+", sep in "[^eE.0-9]", garbage: String) { + let input = format!("{digits}{sep}{garbage}"); + let garbage = format!("{sep}{garbage}"); + prop_assert_eq!( + Number::parse(&input), + Some((Number::Integer { int: Digits(digits.as_str()), exp: None }, garbage.as_str())) + ); + } + + #[test] + fn number_parses_doubles(mut part1 in "[0-9]+", mut part2 in "[0-9]*", swap_parts: bool, e in "[eE]", exponent_sign in "[+-]?", exponent in "[0-9]+", sep in "[^0-9]", garbage: String) { + if swap_parts { + core::mem::swap(&mut part1, &mut part2); + } + let input = format!("{part1}.{part2}{e}{exponent_sign}{exponent}{sep}{garbage}"); + let garbage = format!("{sep}{garbage}"); + let expected = match(part1.is_empty(), part2.is_empty()) { + (false, false) => Number::Decimal { + int: Digits(part1.as_str()), + point: ".", + frac: Digits(part2.as_str()), + exp: Some(Exponent { + e: e.as_str(), + sign: exponent_sign.is_empty().not().then_some(exponent_sign.as_str()), + digits: Digits(exponent.as_str()) + }) + }, + (true, false) => Number::DecimalFraction { + point: ".", + frac: Digits(part2.as_str()), + exp: Some(Exponent { + e: e.as_str(), + sign: exponent_sign.is_empty().not().then_some(exponent_sign.as_str()), + digits: Digits(exponent.as_str()) + }) + }, + (false, true) => Number::DecimalInt { + int: Digits(part1.as_str()), + point: ".", + exp: Some(Exponent { + e: e.as_str(), + sign: exponent_sign.is_empty().not().then_some(exponent_sign.as_str()), + digits: Digits(exponent.as_str()) + }) + }, + (true, true) => unreachable!(), + }; + prop_assert_eq!( + Number::parse(&input), + Some(( + expected, + garbage.as_str() + )) + ); + } + + #[test] + fn not_parses_garbage(input in "[^0-9.]") { + prop_assert_eq!( + Number::parse(&input), + None, + ); + } +} diff --git a/src/lexer_v2/number/tests.rs b/src/lexer_v2/number/tests.rs new file mode 100644 index 0000000..6895bb8 --- /dev/null +++ b/src/lexer_v2/number/tests.rs @@ -0,0 +1,335 @@ +use super::{Digits, Exponent, Number}; + +macro_rules! test_digits_parse { + ($name:ident, $input:literal, $expected:expr) => { + ::paste::paste! { + #[test] + fn [< $name _for_digits >]() { + $crate::log_init(); + + assert_eq!(Digits::parse($input), $expected); + } + } + }; +} + +test_digits_parse!(empty_should_fail, "", None); +test_digits_parse!(whitespace_should_fail, " ", None); +test_digits_parse!(letter_should_fail, "a", None); +test_digits_parse!(minus_should_fail, "-", None); +test_digits_parse!(minus_one_should_fail, "-1", None); + +test_digits_parse!(zero_should_ok, "0", Some((Digits("0"), ""))); +test_digits_parse!(p42_should_ok, "42", Some((Digits("42"), ""))); +test_digits_parse!( + number_space_should_leave_space, + "424233 ", + Some((Digits("424233"), " ")) +); +test_digits_parse!( + number_space_letter_should_leave_space_letter, + "424233 l", + Some((Digits("424233"), " l")) +); +test_digits_parse!( + number_letter_should_leave_letter, + "2p", + Some((Digits("2"), "p")) +); +test_digits_parse!( + big_number, + "9934023759839048593537636598927402740240740720909742094----bbb", + Some(( + Digits("9934023759839048593537636598927402740240740720909742094"), + "----bbb" + )) +); + +macro_rules! test_exponent_parse { + ($name:ident, $input:literal, $expected:expr) => { + ::paste::paste! { + #[test] + fn [< $name _for_exponent >]() { + $crate::log_init(); + + assert_eq!(Exponent::parse($input), $expected); + + } + } + }; +} + +test_exponent_parse!(empty_should_fail, "", None); +test_exponent_parse!(minus_should_fail, "-", None); +test_exponent_parse!(number_should_fail, "423", None); +test_exponent_parse!(only_e_should_fail, "e", None); +test_exponent_parse!(only_cap_e_should_fail, "E", None); + +test_exponent_parse!( + e1_should_ok, + "e1", + Some(( + Exponent { + e: "e", + sign: None, + digits: Digits("1"), + }, + "" + )) +); +test_exponent_parse!( + emore_should_ok, + "e92324l", + Some(( + Exponent { + e: "e", + sign: None, + digits: Digits("92324"), + }, + "l" + )) +); +test_exponent_parse!( + emore_minus_should_ok, + "e-92324l", + Some(( + Exponent { + e: "e", + sign: Some("-"), + digits: Digits("92324"), + }, + "l" + )) +); +test_exponent_parse!( + emore_plus_should_ok, + "e+92324l", + Some(( + Exponent { + e: "e", + sign: Some("+"), + digits: Digits("92324"), + }, + "l" + )) +); +test_exponent_parse!(emore_plus_plus_should_fail, "e++92324l", None); + +test_exponent_parse!( + emore_plus_num_plus_should_ok, + "E+92+", + Some(( + Exponent { + e: "E", + sign: Some("+"), + digits: Digits("92") + }, + "+" + )) +); + +macro_rules! test_number_parse { + ($name:ident, $input:literal, $expected:expr) => { + ::paste::paste! { + #[test] + fn [< $name _for_number >]() { + $crate::log_init(); + + assert_eq!(Number::parse($input), $expected); + } + } + }; +} + +test_number_parse!(empty_should_fail, "", None); +test_number_parse!(word_shold_fail, "word", None); + +test_number_parse!( + int_should_ok, + "1", + Some(( + Number::Integer { + int: Digits("1"), + exp: None + }, + "" + )) +); +test_number_parse!( + int_should_ok2, + "99", + Some(( + Number::Integer { + int: Digits("99"), + exp: None + }, + "" + )) +); + +test_number_parse!( + int_exp, + "99E2", + Some(( + Number::Integer { + int: Digits("99"), + exp: Some(Exponent { + e: "E", + sign: None, + digits: Digits("2") + }) + }, + "" + )) +); + +test_number_parse!( + int_exp_minus, + "99E-2", + Some(( + Number::Integer { + int: Digits("99"), + exp: Some(Exponent { + e: "E", + sign: Some("-"), + digits: Digits("2") + }) + }, + "" + )) +); +test_number_parse!( + int_exp_plus, + "99E+2", + Some(( + Number::Integer { + int: Digits("99"), + exp: Some(Exponent { + e: "E", + sign: Some("+"), + digits: Digits("2") + }) + }, + "" + )) +); +test_number_parse!( + int_plus_plus_should_no_exp, + "99E++2", + Some(( + Number::Integer { + int: Digits("99"), + exp: None + }, + "E++2" + )) +); + +test_number_parse!( + dec_int_only, + "2.", + Some(( + Number::DecimalInt { + int: Digits("2"), + point: ".", + exp: None + }, + "" + )) +); + +test_number_parse!( + dec_int_only_exp, + "2.E-3-", + Some(( + Number::DecimalInt { + int: Digits("2"), + point: ".", + exp: Some(Exponent { + e: "E", + sign: Some("-"), + digits: Digits("3") + }) + }, + "-" + )) +); + +test_number_parse!( + dec_int_full, + "3.141592--", + Some(( + Number::Decimal { + int: Digits("3"), + point: ".", + frac: Digits("141592"), + exp: None + }, + "--" + )) +); + +test_number_parse!( + dec_int_full_exp, + "3141.592e-3+3-", + Some(( + Number::Decimal { + int: Digits("3141"), + point: ".", + frac: Digits("592"), + exp: Some(Exponent { + e: "e", + sign: Some("-"), + digits: Digits("3") + }) + }, + "+3-" + )) +); + +test_number_parse!( + dec_int_frac_exp, + ".592e3+3-", + Some(( + Number::DecimalFraction { + point: ".", + frac: Digits("592"), + exp: Some(Exponent { + e: "e", + sign: None, + digits: Digits("3") + }) + }, + "+3-" + )) +); + +test_number_parse!( + dec_int_frac, + ".592_3+3-", + Some(( + Number::DecimalFraction { + point: ".", + frac: Digits("592"), + exp: None + }, + "_3+3-" + )) +); + +test_number_parse!( + dec_int_frac_exp2, + ".592E-37", + Some(( + Number::DecimalFraction { + point: ".", + frac: Digits("592"), + exp: Some(Exponent { + e: "E", + sign: Some("-"), + digits: Digits("37") + }) + }, + "" + )) +); diff --git a/src/lexer_v2/prop_tests.rs b/src/lexer_v2/prop_tests.rs new file mode 100644 index 0000000..d1ed0ff --- /dev/null +++ b/src/lexer_v2/prop_tests.rs @@ -0,0 +1,17 @@ +use super::{token::SpannedToken, StrTokenParser}; + +use proptest::{prelude::ProptestConfig, proptest}; + +proptest! { + #![proptest_config(ProptestConfig { + cases: 30_000, + max_default_size_range: 500, + ..ProptestConfig::default() + })] + + #[test] + fn parser_never_panics(input: String) { + let parser = StrTokenParser::<'_, SpannedToken<&str>>::new(&input); + let _ = parser.count(); + } +} diff --git a/src/lexer_v2/token/mod.rs b/src/lexer_v2/token/mod.rs new file mode 100644 index 0000000..791ca7a --- /dev/null +++ b/src/lexer_v2/token/mod.rs @@ -0,0 +1,463 @@ +use super::{number::Number, word::Word, Parseable}; + +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)] +pub struct SpannedToken { + token: Token, + span: Span, +} + +impl<'s> Parseable<'s> for SpannedToken<&'s str, &'s str, &'s str, &'s str> { + #[inline] + fn try_parse(input: &'s str) -> Option<(Self, &'s str)> { + Token::try_parse(input).map(|(token, rest)| { + ( + Self { + token, + span: &input[..input.len() - rest.len()], + }, + rest, + ) + }) + } +} + +/// Grouping symbol kind +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, derive_more::Display)] +pub enum GroupingKind { + #[display("paren")] + Parentheses, + #[display("bracket")] + Brackets, + #[display("brace")] + Braces, + #[display("angle")] + Angle, +} + +/// Grouping symbol kind +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, derive_more::Display)] +pub enum GroupingSide { + #[display("left")] + Left, + #[display("right")] + Right, +} + +/// Represents operators (not necessary corresponding to actual [`crate::evaluation_tree::Operator`]s). +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, derive_more::Display)] +pub enum SpecialKind { + #[display(".")] + Dot, + #[display(",")] + Comma, + #[display("+")] + Plus, + #[display("-")] + Minus, + #[display("*")] + Star, + #[display("/")] + Slash, + #[display("%")] + Percent, + #[display("^")] + Cap, + #[display("|")] + Pipe, + #[display("&")] + Umpersand, + #[display(":")] + Colon, + #[display("!")] + Bang, + #[display("=")] + Equals, + #[display("?")] + Question, + #[display("@")] + At, + #[display("~")] + Tilda, +} + +/// Represents a single token. +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, derive_more::Display)] +pub enum Token { + #[display("{kind}_{side}")] + Grouping { + kind: GroupingKind, + side: GroupingSide, + }, + #[display("{_0}")] + Special(SpecialKind), + #[display("{_0}")] + Number(Number), + #[display("{_0}")] + Word(Word), +} + +impl<'s> Parseable<'s> for Token<&'s str, &'s str, &'s str> { + fn try_parse(input: &'s str) -> Option<(Self, &'s str)> { + // if there's no char, there's no token + let c = input.chars().next()?; // (short-circuit) + + // try parsing word + // + // can't be confused with any other token, and mot of the tokens are expected to be words + if let Some((word, rest)) = Word::parse(input) { + return Some((Token::word(word), rest)); + } + + // try parsing number + // + // (same idea) + if let Some((number, rest)) = Number::parse(input) { + return Some((Token::number(number), rest)); + } + + // token is not a number and not a word, so it is a one-character token + let token = match c { + '(' => Token::left_paren(), + ')' => Token::right_paren(), + '[' => Token::left_bracket(), + ']' => Token::right_bracket(), + '{' => Token::left_brace(), + '}' => Token::right_brace(), + '.' => Token::dot(), + ',' => Token::comma(), + '+' => Token::plus(), + '-' => Token::minus(), + '*' => Token::star(), + '/' => Token::slash(), + '%' => Token::percent(), + '^' => Token::cap(), + '|' => Token::pipe(), + '&' => Token::umpersand(), + ':' => Token::colon(), + '!' => Token::bang(), + '=' => Token::equals(), + '?' => Token::question(), + '@' => Token::at(), + '~' => Token::tilda(), + // not a token + _ => return None, + }; + + let next_pos = input + .char_indices() + // position or the char after first + .nth(1) + // or end of the line, if that's the only char + .map_or(input.len(), |(pos, _)| pos); + + Some((token, &input[next_pos..])) + } +} + +// helper constructors and macro +impl Token { + pub fn dot() -> Self { + Token::Special(SpecialKind::Dot) + } + + pub fn comma() -> Self { + Token::Special(SpecialKind::Comma) + } + + pub fn left_paren() -> Self { + Self::Grouping { + kind: GroupingKind::Parentheses, + side: GroupingSide::Left, + } + } + + pub fn right_paren() -> Self { + Self::Grouping { + kind: GroupingKind::Parentheses, + side: GroupingSide::Right, + } + } + + pub fn left_bracket() -> Self { + Self::Grouping { + kind: GroupingKind::Brackets, + side: GroupingSide::Left, + } + } + + pub fn right_bracket() -> Self { + Self::Grouping { + kind: GroupingKind::Brackets, + side: GroupingSide::Right, + } + } + + pub fn left_brace() -> Self { + Self::Grouping { + kind: GroupingKind::Braces, + side: GroupingSide::Left, + } + } + + pub fn right_brace() -> Self { + Self::Grouping { + kind: GroupingKind::Braces, + side: GroupingSide::Right, + } + } + + pub fn left_angle() -> Self { + Self::Grouping { + kind: GroupingKind::Angle, + side: GroupingSide::Left, + } + } + + pub fn right_angle() -> Self { + Self::Grouping { + kind: GroupingKind::Angle, + side: GroupingSide::Right, + } + } + + pub fn plus() -> Self { + Token::Special(SpecialKind::Plus) + } + + pub fn minus() -> Self { + Token::Special(SpecialKind::Minus) + } + + pub fn star() -> Self { + Token::Special(SpecialKind::Star) + } + + pub fn slash() -> Self { + Token::Special(SpecialKind::Slash) + } + + pub fn percent() -> Self { + Token::Special(SpecialKind::Percent) + } + + pub fn cap() -> Self { + Token::Special(SpecialKind::Cap) + } + + pub fn pipe() -> Self { + Token::Special(SpecialKind::Pipe) + } + + pub fn umpersand() -> Self { + Token::Special(SpecialKind::Umpersand) + } + + pub fn colon() -> Self { + Token::Special(SpecialKind::Colon) + } + + pub fn bang() -> Self { + Token::Special(SpecialKind::Bang) + } + + pub fn equals() -> Self { + Token::Special(SpecialKind::Equals) + } + + pub fn question() -> Self { + Token::Special(SpecialKind::Question) + } + + pub fn at() -> Self { + Token::Special(SpecialKind::At) + } + + pub fn tilda() -> Self { + Token::Special(SpecialKind::Tilda) + } + + pub fn number(n: Number) -> Self { + Token::Number(n) + } + + pub fn word(w: Word) -> Self { + Token::Word(w) + } +} + +#[macro_export] +macro_rules! tokens_inner { + () => { + arr![] + }; + ( ( $($inner:tt)* ) $($rest:tt)* ) => { + Concat::concat( + Concat::concat( + Concat::concat( + arr![Token::left_paren()], + tokens_inner!($($inner)*) + ), + arr![Token::right_paren()], + ), + tokens_inner!($($rest)*) + ) + }; + ( [ $($inner:tt)* ] $($rest:tt)* ) => { + Concat::concat( + Concat::concat( + Concat::concat( + arr![Token::left_bracket()], + tokens_inner!($($inner)*) + ), + arr![Token::right_bracket()], + ), + tokens_inner!($($rest)*) + ) + }; + ( { $($inner:tt)* } $($rest:tt)* ) => { + Concat::concat( + Concat::concat( + Concat::concat( + arr![Token::left_brace()], + tokens_inner!($($inner)*) + ), + arr![Token::right_brace()], + ), + tokens_inner!($($rest)*) + ) + }; + (. $($rest:tt)*) => { + Concat::concat( + arr![Token::dot()], + tokens_inner!($($rest)*) + ) + }; + (, $($rest:tt)*) => { + Concat::concat( + arr![Token::comma()], + tokens_inner!($($rest)*) + ) + }; + (+ $($rest:tt)*) => { + Concat::concat( + arr![Token::plus()], + tokens_inner!($($rest)*) + ) + }; + (- $($rest:tt)*) => { + Concat::concat( + arr![Token::minus()], + tokens_inner!($($rest)*) + ) + }; + (* $($rest:tt)*) => { + Concat::concat( + arr![Token::star()], + tokens_inner!($($rest)*) + ) + }; + (/ $($rest:tt)*) => { + Concat::concat( + arr![Token::slash()], + tokens_inner!($($rest)*) + ) + }; + (% $($rest:tt)*) => { + Concat::concat( + arr![Token::percent()], + tokens_inner!($($rest)*) + ) + }; + (^ $($rest:tt)*) => { + Concat::concat( + arr![Token::cap()], + tokens_inner!($($rest)*) + ) + }; + (| $($rest:tt)*) => { + Concat::concat( + arr![Token::pipe()], + tokens_inner!($($rest)*) + ) + }; + (& $($rest:tt)*) => { + Concat::concat( + arr![Token::umpersand()], + tokens_inner!($($rest)*) + ) + }; + (: $($rest:tt)*) => { + Concat::concat( + arr![Token::colon()], + tokens_inner!($($rest)*) + ) + }; + (! $($rest:tt)*) => { + Concat::concat( + arr![Token::bang()], + tokens_inner!($($rest)*) + ) + }; + (= $($rest:tt)*) => { + Concat::concat( + arr![Token::equals()], + tokens_inner!($($rest)*) + ) + }; + (? $($rest:tt)*) => { + Concat::concat( + arr![Token::question()], + tokens_inner!($($rest)*) + ) + }; + (@ $($rest:tt)*) => { + Concat::concat( + arr![Token::at()], + tokens_inner!($($rest)*) + ) + }; + (~ $($rest:tt)*) => { + Concat::concat( + arr![Token::tilda()], + tokens_inner!($($rest)*) + ) + }; + (< $($rest:tt)*) => { + Concat::concat( + arr![Token::left_angle()], + tokens_inner!($($rest)*) + ) + }; + (> $($rest:tt)*) => { + Concat::concat( + arr![Token::right_angle()], + tokens_inner!($($rest)*) + ) + }; + ($token:ident $($rest:tt)*) => { + Concat::concat( + arr![Token::word(word!($token))], + tokens_inner!($($rest)*) + ) + }; + ($token:literal $($rest:tt)*) => { + Concat::concat( + arr![Token::number(number!($token))], + tokens_inner!($($rest)*) + ) + }; +} + +#[macro_export] +macro_rules! tokens { + ($($rest:tt)*) => {{ + #[allow(unused_imports)] + use generic_array::{arr, sequence::Concat}; + #[allow(unused_imports)] + use $crate::{lexer_v2::token::Token, word, tokens_inner, number}; + tokens_inner!($($rest)*) + }}; +} + +#[cfg(test)] +mod tests; diff --git a/src/lexer_v2/token/tests.rs b/src/lexer_v2/token/tests.rs new file mode 100644 index 0000000..a3b4a3d --- /dev/null +++ b/src/lexer_v2/token/tests.rs @@ -0,0 +1,164 @@ +use super::super::StrTokenParser; +use super::Token; +use crate::{number, word}; + +macro_rules! test_tokens { + ($(#[$($attr:tt)+])* $name:ident, ($($input:tt)*), [$($expected:expr),*]) => { + ::paste::paste! { + #[test] + $( + #[$($attr)*] + )* + fn [< $name _tokens >]() { + $crate::log_init(); + + let input = tokens!($($input)*); + let expected = generic_array::GenericArray::, _>::from_array([$($expected),*]); + + assert_eq!(input, expected); + } + } + }; +} + +test_tokens!(empty, (), []); + +test_tokens!(dot, (.), [Token::dot()]); +test_tokens!(comma, (,), [Token::comma()]); +test_tokens!(plus, (+), [Token::plus()]); +test_tokens!(minus, (-), [Token::minus()]); +test_tokens!(star, (*), [Token::star()]); +test_tokens!(slash, (/), [Token::slash()]); +test_tokens!(percent, (%), [Token::percent()]); +test_tokens!(cap, (^), [Token::cap()]); +test_tokens!(pipe, (|), [Token::pipe()]); +test_tokens!(umpersand, (&), [Token::umpersand()]); +test_tokens!(colon, (:), [Token::colon()]); +test_tokens!(bang, (!), [Token::bang()]); +test_tokens!(equals, (=), [Token::equals()]); +test_tokens!(question, (?), [Token::question()]); +test_tokens!(at, (@), [Token::at()]); +test_tokens!(tilda, (~), [Token::tilda()]); +test_tokens!(left_angle, (<), [Token::left_angle()]); +test_tokens!(right_angle, (>), [Token::right_angle()]); + +test_tokens!(word, (word), [Token::word(word!(word))]); +test_tokens!(word2, (w0r3d), [Token::word(word!(w0r3d))]); +test_tokens!( + long_word, + (SOME_REALLY_LONG_WORD_1111111), + [Token::word(word!(SOME_REALLY_LONG_WORD_1111111))] +); +test_tokens!( + #[should_panic = "Invalid word"] + invalid_word, + (r#not_a_valid_word), + [Token::word(word!(not_a_valid_word))] +); + +test_tokens!(number, (1), [Token::number(number!(1))]); +test_tokens!(number2, (1.), [Token::number(number!(1.))]); +test_tokens!(number3, (1.), [Token::number(number!(1.))]); +test_tokens!(number4, (1.0E0), [Token::number(number!(1.0E0))]); + +test_tokens!( + negative_number, + (-24.003E6), + [Token::minus(), Token::number(number!(24.003E6))] +); + +test_tokens!( + very_much_positive_number, + (+ + + 42.0E-5), + [Token::plus(), Token::plus(), Token::plus(), Token::number(number!(42.0E-5))] +); + +test_tokens!( + random_expr, + (3 + 4 - 6 + twelve - 00 - ONLY), + [ + Token::number(number!(3)), + Token::plus(), + Token::number(number!(4)), + Token::minus(), + Token::number(number!(6)), + Token::plus(), + Token::word(word!(twelve)), + Token::minus(), + Token::number(number!(00)), + Token::minus(), + Token::word(word!(ONLY)) + ] +); + +test_tokens!( + parens, + ((2 + 3.4e-21) | [a - _BB45] {} / 10), + [ + Token::left_paren(), + Token::number(number!(2)), + Token::plus(), + Token::number(number!(3.4e-21)), + Token::right_paren(), + Token::pipe(), + Token::left_bracket(), + Token::word(word!(a)), + Token::minus(), + Token::word(word!(_BB45)), + Token::right_bracket(), + Token::left_brace(), + Token::right_brace(), + Token::slash(), + Token::number(number!(10)) + ] +); + +macro_rules! test_token_parser { + ($name:ident, $input:literal, [$($token:expr),*$(,)?]) => { + ::paste::paste! { + #[test] + fn [< $name _tokens_iter >]() { + $crate::log_init(); + + let iter = StrTokenParser::<'_, Token>::new($input); + + let parsed_tokens = iter.collect::>(); + + assert_eq!(parsed_tokens.as_slice(), &[$($token),*]); + } + } + }; +} + +test_token_parser!(empty_parses, "", []); +test_token_parser!(plus_pares, "+", [Ok(Token::plus())]); +test_token_parser!( + expr_parses, + "+3 - b4_A[AA/ #iim 3e-67+ (17.00E-67c ]}- an mee#34\nb) \t ---", + [ + Ok(Token::plus()), + Ok(Token::number(number!(3))), + Ok(Token::minus()), + Ok(Token::word(word!(b4_A))), + Ok(Token::left_bracket()), + Ok(Token::word(word!(AA))), + Ok(Token::slash()), + Err("#iim"), + Ok(Token::number(number!(3e-67))), + Ok(Token::plus()), + Ok(Token::left_paren()), + Ok(Token::number(number!(17.00E-67))), + Ok(Token::word(word!(c))), + Ok(Token::right_bracket()), + Ok(Token::right_brace()), + Ok(Token::minus()), + Ok(Token::word(word!(an))), + Ok(Token::word(word!(mee))), + Err("#34"), + Ok(Token::word(word!(b))), + Ok(Token::right_paren()), + Ok(Token::minus()), + Ok(Token::minus()), + Ok(Token::minus()), + ] +); diff --git a/src/lexer_v2/word/mod.rs b/src/lexer_v2/word/mod.rs new file mode 100644 index 0000000..2f4300b --- /dev/null +++ b/src/lexer_v2/word/mod.rs @@ -0,0 +1,70 @@ +use std::ops::Not; + +#[derive( + Debug, + Clone, + Copy, + Hash, + PartialEq, + Eq, + derive_more::AsRef, + derive_more::Deref, + derive_more::Display, +)] +#[display("{_0}")] +pub struct Word(W); + +#[inline] +fn valid_start(ch: char) -> bool { + matches!(ch, 'a'..='z' | 'A'..='Z' | '_') +} + +#[inline] +fn valid_continue(ch: char) -> bool { + matches!(ch, 'a'..='z' | 'A'..='Z' | '_' | '0'..='9') +} + +impl<'s> Word<&'s str> { + pub fn parse(input: &'s str) -> Option<(Self, &'s str)> { + let mut char_inds = input + // be a good (cat)boy and iterate over characters + .char_indices(); + // get first non-whitespace char (if there's none, we are done) + let (_, first_char) = char_inds.next()?; + if !valid_start(first_char) { + // first non-whitespace is not a valid started for word; bail + return None; + } + // word will end, once we + let end_pos = char_inds + // find next first char that is not a valid continue + .find_map(|(pos, ch)| valid_continue(ch).not().then_some(pos)) + // if there none, word is up to the end of string + .unwrap_or(input.len()); + Some((Self(&input[..end_pos]), &input[end_pos..])) + } +} + +#[macro_export] +macro_rules! word { + (? $input:ident) => { + 'word: { + let Some((word, rest)) = $crate::lexer_v2::word::Word::parse(stringify!($input)) else { + break 'word None; + }; + if !rest.trim().is_empty() { + break 'word None; + } + Some(word) + } + }; + ($input:ident) => { + word!(?$input).expect("Invalid word") + }; +} + +#[cfg(test)] +mod tests; + +#[cfg(test)] +mod prop_tests; diff --git a/src/lexer_v2/word/prop_tests.rs b/src/lexer_v2/word/prop_tests.rs new file mode 100644 index 0000000..8fecec5 --- /dev/null +++ b/src/lexer_v2/word/prop_tests.rs @@ -0,0 +1,35 @@ +use super::Word; + +use proptest::{prelude::ProptestConfig, prop_assert_eq, proptest}; + +proptest! { + #![proptest_config(ProptestConfig { + cases: 10_000, + max_default_size_range: 200, + ..ProptestConfig::default() + })] + + #[test] + fn never_panics(input: String) { + let _ = Word::parse(&input); + } + + #[test] + fn parses_words(word in "[a-zA-Z_][a-zA-Z0-9_]*", sep in "[^a-zA-Z0-9_]", garbage: String) { + let input = format!("{word}{sep}{garbage}"); + let garbage = format!("{sep}{garbage}"); + + prop_assert_eq!( + Word::parse(&input), + Some((Word(word.as_str()), garbage.as_str())) + ); + } + + #[test] + fn not_parses_garbage(input in "[^a-zA-Z_]*") { + prop_assert_eq!( + Word::parse(&input), + None, + ); + } +} diff --git a/src/lexer_v2/word/tests.rs b/src/lexer_v2/word/tests.rs new file mode 100644 index 0000000..ed80e19 --- /dev/null +++ b/src/lexer_v2/word/tests.rs @@ -0,0 +1,36 @@ +use super::Word; + +macro_rules! test_word_parse { + ($name:ident, $input:literal, $expected:expr) => { + #[test] + fn $name() { + assert_eq!(Word::parse($input), $expected); + } + }; +} + +test_word_parse!(empty_should_fail, "", None); +test_word_parse!(number_should_fail, "1", None); +test_word_parse!(whitespace_should_fail, " ", None); +test_word_parse!(non_ascii_should_fail, "слово", None); +test_word_parse!(minus_should_fail, "-", None); + +test_word_parse!(letter_should_ok, "a", Some((Word("a"), ""))); +test_word_parse!(letter_should_ok2, "U", Some((Word("U"), ""))); +test_word_parse!(underscore_should_ok, "_", Some((Word("_"), ""))); +test_word_parse!( + letter_space_should_leave_space, + "a ", + Some((Word("a"), " ")) +); +test_word_parse!( + letter_something_should_keep_something, + "a something", + Some((Word("a"), " something")) +); + +test_word_parse!( + a_minus_b_should_leave_minus_b, + "a-b", + Some((Word("a"), "-b")) +); diff --git a/src/lib.rs b/src/lib.rs index fbe8e80..ebeb518 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,50 @@ #![doc = include_str!("../README.md")] +use core::fmt::Display; + pub mod evaluation_tree; pub mod lexer; pub mod parser; +pub mod lexer_v2; + +#[macro_use] +extern crate tracing; + +#[cfg(test)] +#[allow(unused)] +pub(crate) fn log_init() { + use tracing::level_filters::LevelFilter; + use tracing_subscriber::fmt::{self, format::FmtSpan, time::OffsetTime}; + + #[cfg(verbose_logging)] + if fmt::fmt() + .with_max_level(LevelFilter::TRACE) + .with_span_events(FmtSpan::ACTIVE) + .with_line_number(true) + .with_level(true) + .with_target(true) + .with_timer(OffsetTime::local_rfc_3339().expect("Failed to get local timer")) + .compact() + .try_init() + .is_ok() + { + info!("Log up!"); + } else { + eprintln!("Log is up?"); + } +} + +#[allow( + clippy::ref_option, + reason = "Use case. Don't want to call `.as_ref()` on each site" +)] +pub(crate) fn display_opt(t: &Option) -> &dyn Display { + if let Some(t) = t.as_ref() { + t as &dyn Display + } else { + &"" as &dyn Display + } +} +#[cfg(test)] +extern crate alloc; From d70e25cc5ecaa432aef791087465072d4c129f0a Mon Sep 17 00:00:00 2001 From: Dzuchun Date: Mon, 13 Jan 2025 03:12:17 +0200 Subject: [PATCH 4/8] Made `std` optional. --- Cargo.lock | 30 ++++++++++++++++++++++++++++++ Cargo.toml | 4 ++-- src/evaluation_tree/args.rs | 14 ++++++++------ src/evaluation_tree/arithmetic.rs | 2 ++ src/evaluation_tree/functions.rs | 1 + src/evaluation_tree/mod.rs | 2 ++ src/lexer.rs | 11 ++++++++--- src/lexer_v2/number/mod.rs | 3 +-- src/lexer_v2/number/prop_tests.rs | 2 ++ src/lexer_v2/prop_tests.rs | 2 ++ src/lexer_v2/token/tests.rs | 2 ++ src/lexer_v2/word/mod.rs | 2 +- src/lexer_v2/word/prop_tests.rs | 2 ++ src/lib.rs | 8 +++++++- src/parser.rs | 9 ++++++++- 15 files changed, 78 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a9223be..f09dc27 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "autocfg" version = "1.4.0" @@ -71,6 +77,12 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "errno" version = "0.3.10" @@ -93,6 +105,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0d2fde1f7b3d48b8395d5f2de76c18a528bd6a9cdde438df747bfcba3e05d6f" + [[package]] name = "generic-array" version = "1.2.0" @@ -113,6 +131,17 @@ dependencies = [ "wasi", ] +[[package]] +name = "hashbrown" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + [[package]] name = "itoa" version = "1.0.14" @@ -143,6 +172,7 @@ version = "0.3.0" dependencies = [ "derive_more", "generic-array", + "hashbrown", "nom", "num", "paste", diff --git a/Cargo.toml b/Cargo.toml index 2247cea..b790378 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,7 @@ num = "0.4.3" thiserror = "2.0.11" generic-array = "1.1.1" tracing = { version = "0.1.41", default-features = false, features = ["attributes"] } +hashbrown = "0.15.2" [dev-dependencies] paste = "1.0.15" @@ -20,8 +21,7 @@ tracing-subscriber = { version = "0.3.19", default-features = false, features = proptest = "1.6.0" [features] -default = ["std"] -std = [] +default = [] [lints.rust] unexpected_cfgs = { level = "warn", check-cfg = ['cfg(verbose_logging)'] } diff --git a/src/evaluation_tree/args.rs b/src/evaluation_tree/args.rs index 2ed55a1..68cbfb7 100644 --- a/src/evaluation_tree/args.rs +++ b/src/evaluation_tree/args.rs @@ -16,14 +16,16 @@ //! //! Variables/functions can be unregistered too. Although, there's not much use for it apart from debugging. -use std::{ - collections::{HashMap, HashSet}, - fmt::Debug, -}; +use core::fmt::Debug; + +use alloc::{borrow::ToOwned, boxed::Box, string::String, vec::Vec}; use num::complex::Complex64; + use thiserror::Error; +use hashbrown::{HashMap, HashSet}; + /// Represents types of arguments. /// /// - [`ArgType::Variable`] stands for regular complex variable.s @@ -75,7 +77,7 @@ impl AsRef for ArgsErased { pub struct ArgsRefErased<'l>(pub &'l Args); impl Debug for ArgsRefErased<'_> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { fn display_map(map: &HashMap) -> String { format!( "{{{}}}", @@ -94,7 +96,7 @@ impl Debug for ArgsRefErased<'_> { } impl Debug for ArgsErased { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { ArgsRefErased(&self.0).fmt(f) } } diff --git a/src/evaluation_tree/arithmetic.rs b/src/evaluation_tree/arithmetic.rs index 4ad079e..bb5532a 100644 --- a/src/evaluation_tree/arithmetic.rs +++ b/src/evaluation_tree/arithmetic.rs @@ -27,6 +27,7 @@ impl Operator for Negation { #[cfg(test)] mod negation_tests { + use alloc::borrow::ToOwned; use num::complex::Complex64; use crate::evaluation_tree::{ @@ -111,6 +112,7 @@ impl Operator for Addition { #[cfg(test)] mod addition_tests { + use alloc::borrow::ToOwned; use num::complex::Complex64; use crate::evaluation_tree::{ diff --git a/src/evaluation_tree/functions.rs b/src/evaluation_tree/functions.rs index 512b3a1..c0e9b5d 100644 --- a/src/evaluation_tree/functions.rs +++ b/src/evaluation_tree/functions.rs @@ -6,6 +6,7 @@ //! //! In presented API, only [`NamedFunction`]s are used for now. Although, I might add lots of default functions that will automatically resolve to their [`Functional`] counterparts during parsing. +use alloc::string::String; use num::complex::Complex64; use super::{ diff --git a/src/evaluation_tree/mod.rs b/src/evaluation_tree/mod.rs index 938ad2f..0f334c7 100644 --- a/src/evaluation_tree/mod.rs +++ b/src/evaluation_tree/mod.rs @@ -6,6 +6,7 @@ //! [`EvaluationTree`] can be created by parsing with [`EvaluationTree::from_tokens`] function. //! If you need some custom structure, [`EvaluationTree`] can be ignored and bare [`Node`]s used instead, as they both implement [`Evaluatable`] anyway. +use alloc::{boxed::Box, string::String}; use num::{complex::Complex64, Complex}; use crate::{ @@ -182,6 +183,7 @@ where #[cfg(test)] mod tests { + use alloc::borrow::ToOwned; use num::complex::Complex64; use crate::evaluation_tree::args::{Args, MissingError}; diff --git a/src/lexer.rs b/src/lexer.rs index 41dcc7f..8f02d10 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -1,5 +1,10 @@ //! This module defines lexer and structs relevant to it. +use alloc::{ + borrow::ToOwned, + string::{String, ToString}, + vec::Vec, +}; use nom::{ branch::alt, bytes::complete::tag, @@ -11,8 +16,6 @@ use nom::{ sequence::{pair, preceded}, }; -use core::str::FromStr; - use crate::{ display_opt, lexer_v2::{ @@ -21,7 +24,9 @@ use crate::{ }, }; -type Res<'l, T = Token> = nom::IResult<&'l str, T, Error<&'l str>>; +use core::{fmt::Display, str::FromStr}; + +type Res<'l, T = OldToken> = nom::IResult<&'l str, T, Error<&'l str>>; /// Represents type of grouping. #[derive(Debug, PartialEq, Clone)] diff --git a/src/lexer_v2/number/mod.rs b/src/lexer_v2/number/mod.rs index c4d0a4a..1460bdb 100644 --- a/src/lexer_v2/number/mod.rs +++ b/src/lexer_v2/number/mod.rs @@ -1,5 +1,4 @@ -use core::fmt::Display; -use std::ops::Not; +use core::{fmt::Display, ops::Not}; use crate::display_opt; diff --git a/src/lexer_v2/number/prop_tests.rs b/src/lexer_v2/number/prop_tests.rs index 54444ed..4e14b2b 100644 --- a/src/lexer_v2/number/prop_tests.rs +++ b/src/lexer_v2/number/prop_tests.rs @@ -4,6 +4,8 @@ use super::{Digits, Exponent, Number}; use proptest::{prelude::ProptestConfig, prop_assert_eq, proptest}; +use alloc::string::String; + proptest! { #![proptest_config(ProptestConfig { cases: 10_000, diff --git a/src/lexer_v2/prop_tests.rs b/src/lexer_v2/prop_tests.rs index d1ed0ff..f42ffaf 100644 --- a/src/lexer_v2/prop_tests.rs +++ b/src/lexer_v2/prop_tests.rs @@ -2,6 +2,8 @@ use super::{token::SpannedToken, StrTokenParser}; use proptest::{prelude::ProptestConfig, proptest}; +use alloc::string::String; + proptest! { #![proptest_config(ProptestConfig { cases: 30_000, diff --git a/src/lexer_v2/token/tests.rs b/src/lexer_v2/token/tests.rs index a3b4a3d..712221b 100644 --- a/src/lexer_v2/token/tests.rs +++ b/src/lexer_v2/token/tests.rs @@ -2,6 +2,8 @@ use super::super::StrTokenParser; use super::Token; use crate::{number, word}; +use alloc::vec::Vec; + macro_rules! test_tokens { ($(#[$($attr:tt)+])* $name:ident, ($($input:tt)*), [$($expected:expr),*]) => { ::paste::paste! { diff --git a/src/lexer_v2/word/mod.rs b/src/lexer_v2/word/mod.rs index 2f4300b..578e0fa 100644 --- a/src/lexer_v2/word/mod.rs +++ b/src/lexer_v2/word/mod.rs @@ -1,4 +1,4 @@ -use std::ops::Not; +use core::ops::Not; #[derive( Debug, diff --git a/src/lexer_v2/word/prop_tests.rs b/src/lexer_v2/word/prop_tests.rs index 8fecec5..3cde72f 100644 --- a/src/lexer_v2/word/prop_tests.rs +++ b/src/lexer_v2/word/prop_tests.rs @@ -2,6 +2,8 @@ use super::Word; use proptest::{prelude::ProptestConfig, prop_assert_eq, proptest}; +use alloc::string::String; + proptest! { #![proptest_config(ProptestConfig { cases: 10_000, diff --git a/src/lib.rs b/src/lib.rs index ebeb518..78631d5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,5 @@ #![doc = include_str!("../README.md")] +#![no_std] use core::fmt::Display; @@ -11,6 +12,10 @@ pub mod lexer_v2; #[macro_use] extern crate tracing; +// #[cfg(test)] // <-- TODO: make optional +#[macro_use] +extern crate alloc; + #[cfg(test)] #[allow(unused)] pub(crate) fn log_init() { @@ -47,4 +52,5 @@ pub(crate) fn display_opt(t: &Option) -> &dyn Display { } } #[cfg(test)] -extern crate alloc; +#[macro_use] +extern crate std; diff --git a/src/parser.rs b/src/parser.rs index 015554c..d9d7769 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -1,6 +1,11 @@ //! This module defines parser and relevant structs. -use std::num::NonZeroUsize; +use core::num::NonZeroUsize; + +use alloc::{ + string::{String, ToString}, + vec::Vec, +}; use nom::{ branch::alt, @@ -9,6 +14,7 @@ use nom::{ sequence::tuple, Err, Needed, }; + use num::complex::Complex64; use thiserror::Error; @@ -407,6 +413,7 @@ fn paren(tokens: &[Token]) -> Res<&[Token]> { #[cfg(test)] mod tests { + use alloc::borrow::ToOwned; use num::complex::Complex64; use rand::Rng; From 02147c4aee9e55a852ebf160de69f9e74e8e89d4 Mon Sep 17 00:00:00 2001 From: Dzuchun Date: Mon, 13 Jan 2025 03:59:26 +0200 Subject: [PATCH 5/8] Old-New tokens intertop --- src/evaluation_tree/mod.rs | 4 +- src/lexer.rs | 68 +++++----- src/parser.rs | 258 ++++++++++++++++++------------------- 3 files changed, 162 insertions(+), 168 deletions(-) diff --git a/src/evaluation_tree/mod.rs b/src/evaluation_tree/mod.rs index 0f334c7..edf5518 100644 --- a/src/evaluation_tree/mod.rs +++ b/src/evaluation_tree/mod.rs @@ -10,7 +10,7 @@ use alloc::{boxed::Box, string::String}; use num::{complex::Complex64, Complex}; use crate::{ - lexer::Token, + lexer::OldToken, parser::{parse, GenerationError}, }; @@ -49,7 +49,7 @@ pub struct EvaluationTree(Node); impl EvaluationTree { /// Attempts to parse a tree from token slice. May fail with [`GenerationError`] to indicate incorrect syntax. - pub fn from_tokens(tokens: &[Token]) -> Result { + pub fn from_tokens(tokens: &[OldToken]) -> Result { Ok(Self(parse(tokens)?)) } } diff --git a/src/lexer.rs b/src/lexer.rs index 8f02d10..9a10e45 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -24,7 +24,7 @@ use crate::{ }, }; -use core::{fmt::Display, str::FromStr}; +use core::str::FromStr; type Res<'l, T = OldToken> = nom::IResult<&'l str, T, Error<&'l str>>; @@ -51,7 +51,7 @@ pub enum Operator { /// Represents a single token. #[derive(Debug, PartialEq)] -pub enum Token { +pub enum OldToken { GroupOpen(GroupingType), GroupClose(GroupingType), Operator(Operator), @@ -61,8 +61,6 @@ pub enum Token { Identifier(String), } -<<<<<<< HEAD -======= #[derive(Debug, Clone, Copy, thiserror::Error)] #[error("Unsupported token: {_0}")] pub struct UnsupportedToken<'s>(Token<&'s str, &'s str, &'s str>); @@ -132,7 +130,6 @@ impl<'s> TryFrom> for OldToken { } } ->>>>>>> 7d0e9b0 (impl app) /// Represents token type (used for debug) #[derive(Debug, PartialEq, derive_more::Display)] pub enum TokenType { @@ -145,7 +142,7 @@ pub enum TokenType { fn paren_open(i: &str) -> Res { map(one_of("([{"), |c| { - Token::GroupOpen(match c { + OldToken::GroupOpen(match c { '(' => GroupingType::Parentheses, '[' => GroupingType::Brackets, '{' => GroupingType::Braces, @@ -156,7 +153,7 @@ fn paren_open(i: &str) -> Res { fn paren_close(i: &str) -> Res { map(one_of(")]}"), |c| { - Token::GroupClose(match c { + OldToken::GroupClose(match c { ')' => GroupingType::Parentheses, ']' => GroupingType::Brackets, '}' => GroupingType::Braces, @@ -167,7 +164,7 @@ fn paren_close(i: &str) -> Res { fn operator(i: &str) -> Res { map(one_of("+-*/^"), |c| { - Token::Operator(match c { + OldToken::Operator(match c { '+' => Operator::Plus, '-' => Operator::Minus, '*' => Operator::Star, @@ -179,11 +176,11 @@ fn operator(i: &str) -> Res { } fn comma(i: &str) -> Res { - map(pchar(','), |_| Token::Comma)(i) + map(pchar(','), |_| OldToken::Comma)(i) } fn imaginary_unit(i: &str) -> Res { - map(pchar('i'), |_| Token::ImaginaryUnit)(i) + map(pchar('i'), |_| OldToken::ImaginaryUnit)(i) } // source: https://github.com/rust-bakery/nom/blob/main/doc/nom_recipes.md#identifiers @@ -193,12 +190,12 @@ fn identifier(i: &str) -> Res { alt((alpha1, tag("_"))), many0_count(alt((alphanumeric1, tag("_")))), )), - |name: &str| Token::Identifier(name.to_owned()), + |name: &str| OldToken::Identifier(name.to_owned()), )(i) } fn number(i: &str) -> Res { - map(double, Token::Number)(i) + map(double, OldToken::Number)(i) } fn whitespace(i: &str) -> Res<()> { @@ -206,7 +203,7 @@ fn whitespace(i: &str) -> Res<()> { } /// lexes the input -pub fn lex(i: &str) -> Res> { +pub fn lex(i: &str) -> Res> { fold_many1( preceded( whitespace, @@ -230,18 +227,15 @@ pub fn lex(i: &str) -> Res> { #[cfg(test)] mod tests { -<<<<<<< HEAD -======= use alloc::borrow::ToOwned; use alloc::vec::Vec; ->>>>>>> 7d0e9b0 (impl app) use crate::lexer::GroupingType; use crate::lexer_v2::token::Token; use crate::lexer_v2::StrTokenParser; use super::lex; - use super::Token; + use super::OldToken; #[test] fn tokens() { @@ -257,26 +251,26 @@ mod tests { Ok(( "", vec![ - Token::Number(2f64), - Token::Operator(crate::lexer::Operator::Plus), - Token::Identifier("x".to_owned()), - Token::Operator(crate::lexer::Operator::Cap), - Token::Number(2f64), - Token::Operator(crate::lexer::Operator::Minus), - Token::GroupOpen(GroupingType::Parentheses), - Token::Number(3f64), - Token::Identifier("x".to_owned()), - Token::Operator(crate::lexer::Operator::Cap), - Token::Number(2f64), - Token::Operator(crate::lexer::Operator::Minus), - Token::Number(7f64), - Token::Identifier("y".to_owned()), - Token::Operator(crate::lexer::Operator::Slash), - Token::Identifier("sin".to_owned()), - Token::GroupOpen(GroupingType::Brackets), - Token::Identifier("z".to_owned()), - Token::GroupClose(GroupingType::Brackets), - Token::GroupClose(GroupingType::Parentheses), + OldToken::Number(2f64), + OldToken::Operator(crate::lexer::Operator::Plus), + OldToken::Identifier("x".to_owned()), + OldToken::Operator(crate::lexer::Operator::Cap), + OldToken::Number(2f64), + OldToken::Operator(crate::lexer::Operator::Minus), + OldToken::GroupOpen(GroupingType::Parentheses), + OldToken::Number(3f64), + OldToken::Identifier("x".to_owned()), + OldToken::Operator(crate::lexer::Operator::Cap), + OldToken::Number(2f64), + OldToken::Operator(crate::lexer::Operator::Minus), + OldToken::Number(7f64), + OldToken::Identifier("y".to_owned()), + OldToken::Operator(crate::lexer::Operator::Slash), + OldToken::Identifier("sin".to_owned()), + OldToken::GroupOpen(GroupingType::Brackets), + OldToken::Identifier("z".to_owned()), + OldToken::GroupClose(GroupingType::Brackets), + OldToken::GroupClose(GroupingType::Parentheses), ] )) ); diff --git a/src/parser.rs b/src/parser.rs index d9d7769..2673591 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -24,7 +24,7 @@ use crate::{ functions::NamedFunction, Constant, Node, Variable, }, - lexer::{Operator, Token, TokenType}, + lexer::{OldToken, Operator, TokenType}, }; /// Represents error occurred during tree generation. @@ -32,13 +32,13 @@ use crate::{ pub enum GenerationError<'l> { /// Error emitted by `nom` #[error("{0:?}")] - NomError(NError<&'l [Token]>), + NomError(NError<&'l [OldToken]>), /// There are no more tokens, but some were expected #[error("No tokens")] Empty(Needed), /// Unexpected token type encountered #[error("Bad token: {} expected, but {:?} found", .0, .1)] - BadToken(TokenType, &'l Token), + BadToken(TokenType, &'l OldToken), /// There's an unpaired grouping #[error("Paren was left unclosed")] UnpairedGrouping, @@ -47,12 +47,12 @@ pub enum GenerationError<'l> { NoToken, } -impl<'l> ParseError<&'l [Token]> for GenerationError<'l> { - fn from_error_kind(input: &'l [Token], kind: nom::error::ErrorKind) -> Self { +impl<'l> ParseError<&'l [OldToken]> for GenerationError<'l> { + fn from_error_kind(input: &'l [OldToken], kind: nom::error::ErrorKind) -> Self { Self::NomError(NError::new(input, kind)) } - fn append(_input: &[Token], _kind: nom::error::ErrorKind, other: Self) -> Self { + fn append(_input: &[OldToken], _kind: nom::error::ErrorKind, other: Self) -> Self { // not really sure what this is about, // but it seems that `nom` itself implements this function in a similar fashion, // so I guess it's not much of a harm. @@ -61,10 +61,10 @@ impl<'l> ParseError<&'l [Token]> for GenerationError<'l> { } } -type Res<'l, T = Node> = Result<(&'l [Token], T), Err>>; +type Res<'l, T = Node> = Result<(&'l [OldToken], T), Err>>; /// parses tokens into a node. -pub fn parse(tokens: &[Token]) -> Result, GenerationError> { +pub fn parse(tokens: &[OldToken]) -> Result, GenerationError> { all_consuming(node)(tokens) .map(|(_, n)| n) .map_err(|err| match err { @@ -73,12 +73,12 @@ pub fn parse(tokens: &[Token]) -> Result, GenerationError> { }) } -fn node(tokens: &[Token]) -> Res { +fn node(tokens: &[OldToken]) -> Res { assert_parens(tokens)?; alt((all_consuming(map_parser(paren, raw_node)), raw_node))(tokens) } -fn raw_node(tokens: &[Token]) -> Res { +fn raw_node(tokens: &[OldToken]) -> Res { alt(( all_consuming(function_call), all_consuming(variable), @@ -87,7 +87,7 @@ fn raw_node(tokens: &[Token]) -> Res { ))(tokens) } -fn bin_operator(tokens: &[Token]) -> Res { +fn bin_operator(tokens: &[OldToken]) -> Res { let (_, (node1, op, node2)) = find_bin_operator(tokens)?; let (_, node1) = node(node1)?; let (_, node2) = node(node2)?; @@ -103,7 +103,7 @@ fn bin_operator(tokens: &[Token]) -> Res { )) } -fn find_bin_operator(tokens: &[Token]) -> Res<(&[Token], Operator, &[Token])> { +fn find_bin_operator(tokens: &[OldToken]) -> Res<(&[OldToken], Operator, &[OldToken])> { static PREDESCENCE: [Operator; 5] = [ Operator::Plus, Operator::Minus, @@ -118,9 +118,9 @@ fn find_bin_operator(tokens: &[Token]) -> Res<(&[Token], Operator, &[Token])> { .enumerate() .filter_map(|(i, token)| { match token { - Token::GroupOpen(_) => inner += 1, - Token::GroupClose(_) => inner -= 1, - Token::Operator(op) if op == operator => { + OldToken::GroupOpen(_) => inner += 1, + OldToken::GroupClose(_) => inner -= 1, + OldToken::Operator(op) if op == operator => { if inner == 0 { return Some(i); } @@ -141,7 +141,7 @@ fn find_bin_operator(tokens: &[Token]) -> Res<(&[Token], Operator, &[Token])> { Err(Err::Error(GenerationError::NoToken)) } -fn constant(tokens: &[Token]) -> Res { +fn constant(tokens: &[OldToken]) -> Res { let (rest, val) = alt(( map( tuple((opt(sign), real_part, sign, imaginary_part)), @@ -171,11 +171,11 @@ fn constant(tokens: &[Token]) -> Res { Ok((rest, Constant(val).into())) } -fn real_part(tokens: &[Token]) -> Res { +fn real_part(tokens: &[OldToken]) -> Res { number(tokens) } -fn imaginary_part(tokens: &[Token]) -> Res { +fn imaginary_part(tokens: &[OldToken]) -> Res { alt(( map(tuple((number, opt(star), imaginary_unit)), |(n, _, ())| n), map(tuple((imaginary_unit, star, number)), |((), (), n)| n), @@ -183,7 +183,7 @@ fn imaginary_part(tokens: &[Token]) -> Res { ))(tokens) } -fn function_call(tokens: &[Token]) -> Res { +fn function_call(tokens: &[OldToken]) -> Res { map( tuple(( opt(sign), @@ -208,7 +208,7 @@ fn function_call(tokens: &[Token]) -> Res { )(tokens) } -fn variable(tokens: &[Token]) -> Res { +fn variable(tokens: &[OldToken]) -> Res { map(tuple((opt(sign), identifier)), |(minus, name)| { if let Some(true) = minus { Negation::new(Variable::new(name)).into() @@ -220,14 +220,14 @@ fn variable(tokens: &[Token]) -> Res { // general tokens -fn assert_parens(tokens: &[Token]) -> Res<()> { +fn assert_parens(tokens: &[OldToken]) -> Res<()> { let mut stack = Vec::new(); for token in tokens { - if let Token::GroupOpen(t) = token { + if let OldToken::GroupOpen(t) = token { stack.push(t); } - if let Token::GroupClose(t) = token { + if let OldToken::GroupClose(t) = token { let Some(&e) = stack.last() else { return Err(Err::Failure(GenerationError::UnpairedGrouping)); }; @@ -244,13 +244,13 @@ fn assert_parens(tokens: &[Token]) -> Res<()> { } } -fn number(tokens: &[Token]) -> Res { +fn number(tokens: &[OldToken]) -> Res { let token = tokens .first() .ok_or(Err::Error(GenerationError::Empty(Needed::Size( NonZeroUsize::try_from(1).expect("1 is not zero"), ))))?; - let Token::Number(val) = token else { + let OldToken::Number(val) = token else { return Err(Err::Error(GenerationError::BadToken( TokenType::Constant, token, @@ -259,13 +259,13 @@ fn number(tokens: &[Token]) -> Res { Ok((&tokens[1..], *val)) } -fn plus(tokens: &[Token]) -> Res<()> { +fn plus(tokens: &[OldToken]) -> Res<()> { let t = tokens .first() .ok_or(Err::Error(GenerationError::Empty(Needed::Size( NonZeroUsize::try_from(1).expect("1 is not zero"), ))))?; - let Token::Operator(Operator::Plus) = t else { + let OldToken::Operator(Operator::Plus) = t else { return Err(Err::Error(GenerationError::BadToken( TokenType::Operator, t, @@ -274,13 +274,13 @@ fn plus(tokens: &[Token]) -> Res<()> { Ok((&tokens[1..], ())) } -fn minus(tokens: &[Token]) -> Res<()> { +fn minus(tokens: &[OldToken]) -> Res<()> { let t = tokens .first() .ok_or(Err::Error(GenerationError::Empty(Needed::Size( NonZeroUsize::try_from(1).expect("1 is not zero"), ))))?; - let Token::Operator(Operator::Minus) = t else { + let OldToken::Operator(Operator::Minus) = t else { return Err(Err::Error(GenerationError::BadToken( TokenType::Operator, t, @@ -289,13 +289,13 @@ fn minus(tokens: &[Token]) -> Res<()> { Ok((&tokens[1..], ())) } -fn star(tokens: &[Token]) -> Res<()> { +fn star(tokens: &[OldToken]) -> Res<()> { let t = tokens .first() .ok_or(Err::Error(GenerationError::Empty(Needed::Size( NonZeroUsize::try_from(1).expect("1 is not zero"), ))))?; - let Token::Operator(Operator::Star) = t else { + let OldToken::Operator(Operator::Star) = t else { return Err(Err::Error(GenerationError::BadToken( TokenType::Operator, t, @@ -304,13 +304,13 @@ fn star(tokens: &[Token]) -> Res<()> { Ok((&tokens[1..], ())) } -fn imaginary_unit(tokens: &[Token]) -> Res<()> { +fn imaginary_unit(tokens: &[OldToken]) -> Res<()> { let t = tokens .first() .ok_or(Err::Error(GenerationError::Empty(Needed::Size( NonZeroUsize::try_from(1).expect("1 is not zero"), ))))?; - let Token::ImaginaryUnit = t else { + let OldToken::ImaginaryUnit = t else { return Err(Err::Error(GenerationError::BadToken( TokenType::Operator, t, @@ -319,13 +319,13 @@ fn imaginary_unit(tokens: &[Token]) -> Res<()> { Ok((&tokens[1..], ())) } -fn identifier(tokens: &[Token]) -> Res { +fn identifier(tokens: &[OldToken]) -> Res { let t = tokens .first() .ok_or(Err::Error(GenerationError::Empty(Needed::Size( NonZeroUsize::try_from(1).expect("1 is not zero"), ))))?; - let Token::Identifier(name) = t else { + let OldToken::Identifier(name) = t else { return Err(Err::Error(GenerationError::BadToken( TokenType::Operator, t, @@ -334,21 +334,21 @@ fn identifier(tokens: &[Token]) -> Res { Ok((&tokens[1..], name.to_string())) } -fn comma_separated(tokens: &[Token]) -> Res<(Node, Node)> { +fn comma_separated(tokens: &[OldToken]) -> Res<(Node, Node)> { let (_, (arg1, arg2)) = find_comma(tokens)?; let (_, arg1) = node(arg1)?; let (_, arg2) = node(arg2)?; Ok((&[], (arg1, arg2))) } -fn find_comma(tokens: &[Token]) -> Res<(&[Token], &[Token])> { +fn find_comma(tokens: &[OldToken]) -> Res<(&[OldToken], &[OldToken])> { let mut inline = Vec::new(); for (i, token) in tokens.iter().enumerate() { match token { - Token::GroupOpen(t) => { + OldToken::GroupOpen(t) => { inline.push(t); } - Token::GroupClose(t) => { + OldToken::GroupClose(t) => { if &t != inline .last() @@ -359,7 +359,7 @@ fn find_comma(tokens: &[Token]) -> Res<(&[Token], &[Token])> { inline.pop(); } - Token::Comma => { + OldToken::Comma => { if inline.is_empty() { return Ok((&[], (&tokens[..i], &tokens[i + 1..]))); } @@ -370,17 +370,17 @@ fn find_comma(tokens: &[Token]) -> Res<(&[Token], &[Token])> { Err(Err::Error(GenerationError::NoToken)) } -fn sign(tokens: &[Token]) -> Res { +fn sign(tokens: &[OldToken]) -> Res { alt((map(plus, |()| false), map(minus, |()| true)))(tokens) } -fn paren(tokens: &[Token]) -> Res<&[Token]> { +fn paren(tokens: &[OldToken]) -> Res<&[OldToken]> { let t = tokens .first() .ok_or(Err::Error(GenerationError::Empty(Needed::Size( NonZeroUsize::try_from(1).expect("1 is not zero"), ))))?; - let Token::GroupOpen(t) = t else { + let OldToken::GroupOpen(t) = t else { return Err(Err::Error(GenerationError::BadToken( TokenType::Grouping, t, @@ -390,12 +390,12 @@ fn paren(tokens: &[Token]) -> Res<&[Token]> { let mut inner = 1; for i in 1..tokens.len() { let token = &tokens[i]; - if let Token::GroupOpen(e) = token { + if let OldToken::GroupOpen(e) = token { if t == e { inner += 1; } } - if let Token::GroupClose(e) = token { + if let OldToken::GroupClose(e) = token { if t == e { inner -= 1; @@ -424,7 +424,7 @@ mod tests { functions::NamedFunction, Constant, Evaluatable, Node, Variable, }, - lexer::{GroupingType, Operator, Token}, + lexer::{GroupingType, OldToken, Operator}, }; use super::parse; @@ -489,122 +489,122 @@ mod tests { // following tests reinforce that complex constant recognition works as expected: test! {constant1, - tokens = [Token::Number(1.0f64)], + tokens = [OldToken::Number(1.0f64)], expected = Constant::from(1.0f64)} test! {constant2, - tokens = [Token::ImaginaryUnit], + tokens = [OldToken::ImaginaryUnit], expected = Constant::from(1.0f64).tr()} test! {constant3, tokens = [ - Token::Number(1.0f64), - Token::Operator(Operator::Star), - Token::ImaginaryUnit + OldToken::Number(1.0f64), + OldToken::Operator(Operator::Star), + OldToken::ImaginaryUnit ], expected = Constant::from(1.0f64).tr()} test! {constant4, tokens = [ - Token::Number(1.0f64), - Token::Operator(Operator::Plus), - Token::ImaginaryUnit + OldToken::Number(1.0f64), + OldToken::Operator(Operator::Plus), + OldToken::ImaginaryUnit ], expected = Constant(Complex64::new(1.0, 1.0))} test! {constant5, tokens = [ - Token::Operator(Operator::Minus), - Token::Number(1.0f64), - Token::Operator(Operator::Plus), - Token::Number(1.0), - Token::Operator(Operator::Star), - Token::ImaginaryUnit + OldToken::Operator(Operator::Minus), + OldToken::Number(1.0f64), + OldToken::Operator(Operator::Plus), + OldToken::Number(1.0), + OldToken::Operator(Operator::Star), + OldToken::ImaginaryUnit ], expected = Constant(Complex64::new(-1.0, 1.0))} test! {constant6, tokens = [ - Token::Operator(Operator::Minus), - Token::Number(1.0f64), - Token::Operator(Operator::Plus), - Token::Number(1.0), Token::ImaginaryUnit + OldToken::Operator(Operator::Minus), + OldToken::Number(1.0f64), + OldToken::Operator(Operator::Plus), + OldToken::Number(1.0), OldToken::ImaginaryUnit ], expected = Constant(Complex64::new(-1.0, 1.0))} test! {constant7, tokens = [ - Token::Operator(Operator::Minus), - Token::ImaginaryUnit, - Token::Operator(Operator::Star), - Token::Number(1.0f64), - Token::Operator(Operator::Plus), - Token::Number(1.0) + OldToken::Operator(Operator::Minus), + OldToken::ImaginaryUnit, + OldToken::Operator(Operator::Star), + OldToken::Number(1.0f64), + OldToken::Operator(Operator::Plus), + OldToken::Number(1.0) ], expected = Constant(Complex64::new(1.0, -1.0))} test! {constant8, tokens = [ - Token::Operator(Operator::Minus), - Token::ImaginaryUnit, - Token::Operator(Operator::Plus), - Token::Number(1.0) + OldToken::Operator(Operator::Minus), + OldToken::ImaginaryUnit, + OldToken::Operator(Operator::Plus), + OldToken::Number(1.0) ], expected = Constant(Complex64::new(1.0, -1.0))} test! {constant9, tokens = [ - Token::Operator(Operator::Minus), - Token::ImaginaryUnit + OldToken::Operator(Operator::Minus), + OldToken::ImaginaryUnit ], expected = Constant(Complex64::new(0.0, -1.0))} // following tests reinforce that variable recognition works as expected: test! {variable1, - tokens = [Token::Identifier("x".to_owned())], + tokens = [OldToken::Identifier("x".to_owned())], expected = Variable::new("x")} test! {variable2, - tokens = [Token::Identifier("a_123".to_owned())], + tokens = [OldToken::Identifier("a_123".to_owned())], expected = Variable::new("a_123")} // following tests reinforce that function calls work as expected: test! {function1, tokens = [ - Token::Identifier("sin".to_owned()), - Token::GroupOpen(GroupingType::Parentheses), - Token::Identifier("x".to_owned()), - Token::GroupClose(GroupingType::Parentheses) + OldToken::Identifier("sin".to_owned()), + OldToken::GroupOpen(GroupingType::Parentheses), + OldToken::Identifier("x".to_owned()), + OldToken::GroupClose(GroupingType::Parentheses) ], expected = NamedFunction::new("sin", Variable::new("x"))} test! {function2, tokens = [ - Token::Identifier("sum_sq".to_owned()), - Token::GroupOpen(GroupingType::Parentheses), - Token::Identifier("x".to_owned()), - Token::Comma, - Token::Identifier("y".to_owned()), - Token::GroupClose(GroupingType::Parentheses) + OldToken::Identifier("sum_sq".to_owned()), + OldToken::GroupOpen(GroupingType::Parentheses), + OldToken::Identifier("x".to_owned()), + OldToken::Comma, + OldToken::Identifier("y".to_owned()), + OldToken::GroupClose(GroupingType::Parentheses) ], expected = NamedFunction::new("sum_sq", (Variable::new("x"), Variable::new("y")))} test! {function3, tokens = [ - Token::Operator(Operator::Minus), - Token::Identifier("sin".to_owned()), - Token::GroupOpen(GroupingType::Parentheses), - Token::Identifier("x".to_owned()), - Token::GroupClose(GroupingType::Parentheses) + OldToken::Operator(Operator::Minus), + OldToken::Identifier("sin".to_owned()), + OldToken::GroupOpen(GroupingType::Parentheses), + OldToken::Identifier("x".to_owned()), + OldToken::GroupClose(GroupingType::Parentheses) ], expected = Negation::new(NamedFunction::new("sin", Variable::new("x")))} // following tests reinforce correct binary operator parsing // x + y * 2 test! {bin_operator1, tokens = [ - Token::Identifier("x".to_owned()), - Token::Operator(Operator::Plus), - Token::Identifier("y".to_owned()), - Token::Operator(Operator::Star), - Token::Number(2f64) + OldToken::Identifier("x".to_owned()), + OldToken::Operator(Operator::Plus), + OldToken::Identifier("y".to_owned()), + OldToken::Operator(Operator::Star), + OldToken::Number(2f64) ], expected = Addition::new(Variable::new("x"), Multiplication::new(Variable::new("y"), Constant(2f64.into())))} // [-x + 4] * sin{y}^2 test! {bin_operator2, tokens = [ - Token::GroupOpen(GroupingType::Brackets), - Token::Operator(Operator::Minus), - Token::Identifier("x".to_owned()), - Token::Operator(Operator::Plus), - Token::Number(4f64), - Token::GroupClose(GroupingType::Brackets), - Token::Operator(Operator::Star), - Token::Identifier("sin".to_owned()), - Token::GroupOpen(GroupingType::Braces), - Token::Identifier("y".to_owned()), - Token::GroupClose(GroupingType::Braces), - Token::Operator(Operator::Cap), - Token::Number(2f64) + OldToken::GroupOpen(GroupingType::Brackets), + OldToken::Operator(Operator::Minus), + OldToken::Identifier("x".to_owned()), + OldToken::Operator(Operator::Plus), + OldToken::Number(4f64), + OldToken::GroupClose(GroupingType::Brackets), + OldToken::Operator(Operator::Star), + OldToken::Identifier("sin".to_owned()), + OldToken::GroupOpen(GroupingType::Braces), + OldToken::Identifier("y".to_owned()), + OldToken::GroupClose(GroupingType::Braces), + OldToken::Operator(Operator::Cap), + OldToken::Number(2f64) ], expected = Multiplication::new( Addition::new( Negation::new(Variable::new("x")), @@ -621,24 +621,24 @@ mod tests { // z ^ [z] + sin(1.0 / e ^ {i * phi}) test! {bin_operator3, tokens = [ - Token::Identifier("z".to_owned()), - Token::Operator(Operator::Cap), - Token::GroupOpen(GroupingType::Brackets), - Token::Identifier("z".to_owned()), - Token::GroupClose(GroupingType::Brackets), - Token::Operator(Operator::Plus), - Token::Identifier("sin".to_owned()), - Token::GroupOpen(GroupingType::Parentheses), - Token::Number(1.0), - Token::Operator(Operator::Slash), - Token::Identifier("e".to_owned()), - Token::Operator(Operator::Cap), - Token::GroupOpen(GroupingType::Braces), - Token::ImaginaryUnit, - Token::Operator(Operator::Star), - Token::Identifier("phi".to_owned()), - Token::GroupClose(GroupingType::Braces), - Token::GroupClose(GroupingType::Parentheses) + OldToken::Identifier("z".to_owned()), + OldToken::Operator(Operator::Cap), + OldToken::GroupOpen(GroupingType::Brackets), + OldToken::Identifier("z".to_owned()), + OldToken::GroupClose(GroupingType::Brackets), + OldToken::Operator(Operator::Plus), + OldToken::Identifier("sin".to_owned()), + OldToken::GroupOpen(GroupingType::Parentheses), + OldToken::Number(1.0), + OldToken::Operator(Operator::Slash), + OldToken::Identifier("e".to_owned()), + OldToken::Operator(Operator::Cap), + OldToken::GroupOpen(GroupingType::Braces), + OldToken::ImaginaryUnit, + OldToken::Operator(Operator::Star), + OldToken::Identifier("phi".to_owned()), + OldToken::GroupClose(GroupingType::Braces), + OldToken::GroupClose(GroupingType::Parentheses) ], expected = Addition::new( Exponentiation::new( Variable::new("z"), From 243b6ad0035c2969a76ab8163ced5dc0fbb04ea1 Mon Sep 17 00:00:00 2001 From: Dzuchun Date: Mon, 13 Jan 2025 04:53:23 +0200 Subject: [PATCH 6/8] Version bump --- Cargo.lock | 2 +- Cargo.toml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f09dc27..a44a3dd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -168,7 +168,7 @@ checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" [[package]] name = "mathvis" -version = "0.3.0" +version = "0.4.0" dependencies = [ "derive_more", "generic-array", diff --git a/Cargo.toml b/Cargo.toml index b790378..c5dfa15 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mathvis" -version = "0.3.0" +version = "0.4.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -10,7 +10,7 @@ derive_more = {version = "1.0.0", features = ["display", "as_ref", "deref"]} nom = "7.1.3" num = "0.4.3" thiserror = "2.0.11" -generic-array = "1.1.1" +generic-array = "1.2.0" tracing = { version = "0.1.41", default-features = false, features = ["attributes"] } hashbrown = "0.15.2" From 9d466d7896610df37207d1c34f7d77a59864e518 Mon Sep 17 00:00:00 2001 From: Dzuchun Date: Sun, 21 Dec 2025 03:21:23 +0000 Subject: [PATCH 7/8] migration notice --- README.md | 175 +----------------------------------------------------- 1 file changed, 1 insertion(+), 174 deletions(-) diff --git a/README.md b/README.md index de689cb..ac29ce9 100644 --- a/README.md +++ b/README.md @@ -1,174 +1 @@ -The name is `MathVis`. Stands for "Math Vision" - -## Adding to project -There's no cargo entry for it yet, but you can (hopefully) still use it by including -```toml - mathvis = { git = "https://github.com/Dzuchun/mathvis.git" } -``` -into your project's `Cargo.toml` file. - -# Overview -This is a simple library I coded to recognize basic math expressions involving constants, variables, arithmetic operations and functions. Operating is done exclusively with `num::complex::Complex64` for now, as I have failed with dynamic typing (would appreciate any help with that, you may see some implementations for arbitrary types at my [`first commit`]). - -Here's some info on what this library can be used for: - -## Lexing -All text input should first be lexed (i.e. turned into tokens) first. This can be done with `mathvis::lexer::lex` function, returning `Vec`, if successful. - -At this point, you may recognize some extended syntax and replace it with your own. Examples: -- `2x` would not be parsed as `2 * x` by default (it would be an error, actually). You may insert `Token::Operator(Operator::Star)` between each `Token::Number` and `Token::Ident` you find to get this sort of behavior. -- Only functions with one or two arguments are supported for now. But three and more - argument function syntax can still recognized my you manually and transformed into two-argument function call(s), if possible. - -## Parsing -Slice of tokens can be parsed (i.e. turned into evaluation tree). This is done with `mathvis::evaluation_tree::EvaluationTree::from_tokens` function, returning `EvaluationTree`, if successful. - -`EvaluationTree` is my way to represent syntax, in a form that's easy to compute. You can't make any modifications into syntax at this point, *but* you still have not defined any variables and/or functions used, as these are not represented by this tree. - -## Evaluation -`EvaluationTree` can be, well, evaluated to a single `Complex64` with `mathvis::evaluation_tree::Evaluatable::evaluate` function. To do that, it needs a suitable `mathvis::evaluation_tree::args::Args` object, used to represent variable values and function definitions. To obtain it (and find out which variables/functions parsed expression actually need), you may use `mathvis::evaluation_tree::Evaluatable::args` function defined for `EvaluationTree`. Resulting `Args` object will have all the required values *registered*, but *assigned*. You are expected to assign all of the values/functions registered by a tree. - -You can define your own constants and functions this way, for example: -- To define a constant, you'll want to unconditionally call `Args::assign_variable`. -- To define a custom function, you'll want to use `Args::assign_function`/`Args::assign_function2`. These accept `impl Fn`s, so you may use function pointers as well as closures. - -There's no problem in case of unused assignments in `Args` struct, so you are *not forced* to use your custom constants/functions. - -Both `EvaluationTree` and `Args` are **not** consumed upon evaluation, meaning you can reuse same tree for iterative computations as well as modify arguments to get a different result. - -# Examples -## Simple -The following is a `simple` example, involving all of the steps described above - -```rust -# use num::complex::Complex64; -# use crate::mathvis::evaluation_tree::Evaluatable; -let input = "2 * sum_sq(x, y) ^ y + sin(x * y)"; - -let (_, tokens) = mathvis::lexer::lex(input).expect("Should be able to recognize tokens"); - -let tree = mathvis::evaluation_tree::EvaluationTree::from_tokens(tokens.as_slice()) - .expect("Should be able to parse into tree"); - -let mut args = tree.args(); -// args are expected to contain variables "x" and "y", as well as "sin" function and "sum_sq" function2 -// let define them, then: -args.assign_variable("x", Complex64::new(2.0, -3.0)); -args.assign_variable("y", Complex64::new(-std::f64::consts::PI, 1.0)); -args.assign_function("sin", Complex64::sin); -args.assign_function2("sum_sq", |x, y| x * x + y * y); - -let result = tree.evaluate(&args).expect("Should be able to evaluate"); - -print!("{}", result); -``` -I got an output of -```bash -6460.656547978857-45323.133349725416i -``` -which is similar to [what WolframAlpha gets][simple wolfram] - -## Cli calculator -The following is a loop part of a `cli_calculator` example. Steps can be clearly seen here. - -```rust,ignore -// read input from stdin -print!("\n\nPlease, input your formula: "); -std::io::stdout().flush().unwrap(); -input.clear(); -if std::io::stdin().read_line(&mut input).unwrap() == 0 { - return; -} -input.pop(); // pop '\n' - -// lex to tokens -let tokens = match mathvis::lexer::lex(input.as_str()) { - Ok((left, tokens)) => { - if !left.is_empty() { - println!("Info: \"{left}\" was ignored"); - } - tokens - } - Err(err) => { - eprintln!("Failed to evaluate: {err}"); - continue 'outer; - } -}; - -// parse to tree -let tree = match mathvis::evaluation_tree::EvaluationTree::from_tokens(tokens.as_slice()) { - Ok(tree) => tree, - Err(err) => { - eprintln!("Failed to parse tokens: {err}"); - continue 'outer; - } -}; - -// create the arguments -let mut args = tree.args(); // input args -args.merge(default_args()); // merge with default args - -if args.functions2().into_iter().next().is_some() { - // there are functions2, this expression will not be evaluated - eprintln!("CLI calculator does not support functions2"); - continue 'outer; -} - -if let Some(name) = args - .functions_mut() - .into_iter() - .find_map(|(name, v)| v.is_none().then_some(name)) -{ - // there are unknown functions - eprintln!("{name} is an unknown function. CLI calculator does not support your own function definition, please use trig functions only"); - continue 'outer; -} - -// ask user to assign all the variables -for (name, val) in args.variables_mut() { - if val.is_some() { - continue; - } - print!("Please assign {name} := "); - std::io::stdout().flush().unwrap(); - input.clear(); - if std::io::stdin().read_line(&mut input).unwrap() == 0 { - return; - } - input.pop(); // pop '\n' - match input.parse() { - Ok(parsed) => { - *val = Some(parsed); - } - Err(err) => { - eprintln!("Bad assignment format: {err}"); - continue 'outer; - } - } -} - -// evaluate -let res = match tree.evaluate(&args) { - Ok(res) => res, - Err(err) => { - eprintln!("Unexpected error while evaluation: {err}"); - continue 'outer; - } -}; - -println!("The end result is {res}"); -``` - -# On complex arithmetic -Pretty much the only this end user does not control about it - is complex exponentiation syntax. It should be noted, that strictly-speaking, this operation would have several results in case of a non-integer exponent. Currently, I'm using `num`'s exponentiation implementation, but you can obvious define your own (just define a function2 for it). - - -# Flaws -- **Lots of allocations**: unfortunately, I see no *safe* way to avoid these on every tree node. -- **Slow computation**: right now tree is actually walked around to compute the result. This is less than ideal, as it results in uncountable many pointer chases, and well as decent recursion leveling. To mitigate this, I plan on creating a module to convert `EvaluationTree` into sort-of-compiled instructions array that can be linearly followed during the execution. Any suggestions on this end would be greatly appreciated too. -- **Traits instead of enums**: for some reason, I decided it would be great to give end-used an ability to define their custom node types. But now, when I think about it, it seems that there are not much to be added, especially considering I do not grant an ability for custom syntax (apart from bare token modifications). - -I'll try to deal with these some time in the future, but it is like that for now, I guess. - - -[`first commit`]: https://github.com/Dzuchun/mathvis/commit/6b9cd1ad46366f689b4afd7a4c35069eead69a0f -[simple wolfram]: https://www.wolframalpha.com/input?i=2+*+%28%282+-+3i%29%5E2+%2B++%28-pi%2Bi%29%5E2%29+%5E+%28-pi%2Bi%29+%2B+sin%28%282+-+3i%29+*+%28-pi%2Bi%29%29 +Migrated to [dzuchun.ing](https://git.dzuchun.ing/dzu/mathvis) From 13458aac119260b857299ffc57e9f206ea739344 Mon Sep 17 00:00:00 2001 From: Dzuchun Date: Sun, 21 Dec 2025 03:23:00 +0000 Subject: [PATCH 8/8] Revert "migration notice" This reverts commit 9d466d7896610df37207d1c34f7d77a59864e518. --- README.md | 175 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 174 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ac29ce9..de689cb 100644 --- a/README.md +++ b/README.md @@ -1 +1,174 @@ -Migrated to [dzuchun.ing](https://git.dzuchun.ing/dzu/mathvis) +The name is `MathVis`. Stands for "Math Vision" + +## Adding to project +There's no cargo entry for it yet, but you can (hopefully) still use it by including +```toml + mathvis = { git = "https://github.com/Dzuchun/mathvis.git" } +``` +into your project's `Cargo.toml` file. + +# Overview +This is a simple library I coded to recognize basic math expressions involving constants, variables, arithmetic operations and functions. Operating is done exclusively with `num::complex::Complex64` for now, as I have failed with dynamic typing (would appreciate any help with that, you may see some implementations for arbitrary types at my [`first commit`]). + +Here's some info on what this library can be used for: + +## Lexing +All text input should first be lexed (i.e. turned into tokens) first. This can be done with `mathvis::lexer::lex` function, returning `Vec`, if successful. + +At this point, you may recognize some extended syntax and replace it with your own. Examples: +- `2x` would not be parsed as `2 * x` by default (it would be an error, actually). You may insert `Token::Operator(Operator::Star)` between each `Token::Number` and `Token::Ident` you find to get this sort of behavior. +- Only functions with one or two arguments are supported for now. But three and more - argument function syntax can still recognized my you manually and transformed into two-argument function call(s), if possible. + +## Parsing +Slice of tokens can be parsed (i.e. turned into evaluation tree). This is done with `mathvis::evaluation_tree::EvaluationTree::from_tokens` function, returning `EvaluationTree`, if successful. + +`EvaluationTree` is my way to represent syntax, in a form that's easy to compute. You can't make any modifications into syntax at this point, *but* you still have not defined any variables and/or functions used, as these are not represented by this tree. + +## Evaluation +`EvaluationTree` can be, well, evaluated to a single `Complex64` with `mathvis::evaluation_tree::Evaluatable::evaluate` function. To do that, it needs a suitable `mathvis::evaluation_tree::args::Args` object, used to represent variable values and function definitions. To obtain it (and find out which variables/functions parsed expression actually need), you may use `mathvis::evaluation_tree::Evaluatable::args` function defined for `EvaluationTree`. Resulting `Args` object will have all the required values *registered*, but *assigned*. You are expected to assign all of the values/functions registered by a tree. + +You can define your own constants and functions this way, for example: +- To define a constant, you'll want to unconditionally call `Args::assign_variable`. +- To define a custom function, you'll want to use `Args::assign_function`/`Args::assign_function2`. These accept `impl Fn`s, so you may use function pointers as well as closures. + +There's no problem in case of unused assignments in `Args` struct, so you are *not forced* to use your custom constants/functions. + +Both `EvaluationTree` and `Args` are **not** consumed upon evaluation, meaning you can reuse same tree for iterative computations as well as modify arguments to get a different result. + +# Examples +## Simple +The following is a `simple` example, involving all of the steps described above + +```rust +# use num::complex::Complex64; +# use crate::mathvis::evaluation_tree::Evaluatable; +let input = "2 * sum_sq(x, y) ^ y + sin(x * y)"; + +let (_, tokens) = mathvis::lexer::lex(input).expect("Should be able to recognize tokens"); + +let tree = mathvis::evaluation_tree::EvaluationTree::from_tokens(tokens.as_slice()) + .expect("Should be able to parse into tree"); + +let mut args = tree.args(); +// args are expected to contain variables "x" and "y", as well as "sin" function and "sum_sq" function2 +// let define them, then: +args.assign_variable("x", Complex64::new(2.0, -3.0)); +args.assign_variable("y", Complex64::new(-std::f64::consts::PI, 1.0)); +args.assign_function("sin", Complex64::sin); +args.assign_function2("sum_sq", |x, y| x * x + y * y); + +let result = tree.evaluate(&args).expect("Should be able to evaluate"); + +print!("{}", result); +``` +I got an output of +```bash +6460.656547978857-45323.133349725416i +``` +which is similar to [what WolframAlpha gets][simple wolfram] + +## Cli calculator +The following is a loop part of a `cli_calculator` example. Steps can be clearly seen here. + +```rust,ignore +// read input from stdin +print!("\n\nPlease, input your formula: "); +std::io::stdout().flush().unwrap(); +input.clear(); +if std::io::stdin().read_line(&mut input).unwrap() == 0 { + return; +} +input.pop(); // pop '\n' + +// lex to tokens +let tokens = match mathvis::lexer::lex(input.as_str()) { + Ok((left, tokens)) => { + if !left.is_empty() { + println!("Info: \"{left}\" was ignored"); + } + tokens + } + Err(err) => { + eprintln!("Failed to evaluate: {err}"); + continue 'outer; + } +}; + +// parse to tree +let tree = match mathvis::evaluation_tree::EvaluationTree::from_tokens(tokens.as_slice()) { + Ok(tree) => tree, + Err(err) => { + eprintln!("Failed to parse tokens: {err}"); + continue 'outer; + } +}; + +// create the arguments +let mut args = tree.args(); // input args +args.merge(default_args()); // merge with default args + +if args.functions2().into_iter().next().is_some() { + // there are functions2, this expression will not be evaluated + eprintln!("CLI calculator does not support functions2"); + continue 'outer; +} + +if let Some(name) = args + .functions_mut() + .into_iter() + .find_map(|(name, v)| v.is_none().then_some(name)) +{ + // there are unknown functions + eprintln!("{name} is an unknown function. CLI calculator does not support your own function definition, please use trig functions only"); + continue 'outer; +} + +// ask user to assign all the variables +for (name, val) in args.variables_mut() { + if val.is_some() { + continue; + } + print!("Please assign {name} := "); + std::io::stdout().flush().unwrap(); + input.clear(); + if std::io::stdin().read_line(&mut input).unwrap() == 0 { + return; + } + input.pop(); // pop '\n' + match input.parse() { + Ok(parsed) => { + *val = Some(parsed); + } + Err(err) => { + eprintln!("Bad assignment format: {err}"); + continue 'outer; + } + } +} + +// evaluate +let res = match tree.evaluate(&args) { + Ok(res) => res, + Err(err) => { + eprintln!("Unexpected error while evaluation: {err}"); + continue 'outer; + } +}; + +println!("The end result is {res}"); +``` + +# On complex arithmetic +Pretty much the only this end user does not control about it - is complex exponentiation syntax. It should be noted, that strictly-speaking, this operation would have several results in case of a non-integer exponent. Currently, I'm using `num`'s exponentiation implementation, but you can obvious define your own (just define a function2 for it). + + +# Flaws +- **Lots of allocations**: unfortunately, I see no *safe* way to avoid these on every tree node. +- **Slow computation**: right now tree is actually walked around to compute the result. This is less than ideal, as it results in uncountable many pointer chases, and well as decent recursion leveling. To mitigate this, I plan on creating a module to convert `EvaluationTree` into sort-of-compiled instructions array that can be linearly followed during the execution. Any suggestions on this end would be greatly appreciated too. +- **Traits instead of enums**: for some reason, I decided it would be great to give end-used an ability to define their custom node types. But now, when I think about it, it seems that there are not much to be added, especially considering I do not grant an ability for custom syntax (apart from bare token modifications). + +I'll try to deal with these some time in the future, but it is like that for now, I guess. + + +[`first commit`]: https://github.com/Dzuchun/mathvis/commit/6b9cd1ad46366f689b4afd7a4c35069eead69a0f +[simple wolfram]: https://www.wolframalpha.com/input?i=2+*+%28%282+-+3i%29%5E2+%2B++%28-pi%2Bi%29%5E2%29+%5E+%28-pi%2Bi%29+%2B+sin%28%282+-+3i%29+*+%28-pi%2Bi%29%29