Merge pull request #175 from TruncateGame/main

Production Truncate Release — 4 February 2024

.gitignore

@@ -5,8 +5,6 @@ web_client/src/static/truncate_client.js
 kaikki*
 node_modules
 word_definitions/package-lock.json
-word_freqs/package-lock.json
-word_freqs/en_word_freqs.txt
 truncate_client/img/package-lock.json
 _site
 # ungzipped word definition db
@@ -15,4 +13,8 @@ local_defs.db
 unsummed_wordlist.txt
 unmarked_wordlist.txt
 objectionable.json
-.env
+.env
+dict_builder/support_data/generated_scowl_wordlists
+dict_builder/support_data/en_word_freqs.txt
+dict_builder/support_data/objectionable.json
+dict_builder/support_data/wordnik_wordlist.txt

Cargo.toml

@@ -1,6 +1,7 @@
 [workspace]
 
 members = [
+    "dict_builder",
     "truncate_core",
     "truncate_server",
     "truncate_client",

Dockerfile.citadel

@@ -21,7 +21,7 @@ ENV SQLX_OFFLINE="true"
 
 ADD truncate_server /app/truncate_server
 ADD truncate_core /app/truncate_core
-ADD word_freqs /app/word_freqs
+ADD dict_builder /app/dict_builder
 
 RUN cd truncate_server && cargo build --release
 

Dockerfile.client

@@ -32,7 +32,7 @@ ADD truncate_server /app/truncate_server
 ADD truncate_client /app/truncate_client
 ADD truncate_dueller /app/truncate_dueller
 ADD truncate_core /app/truncate_core
-ADD word_freqs /app/word_freqs
+ADD dict_builder /app/dict_builder
 ADD Cargo.* /app
 
 ADD web_client /app/web_client

dict_builder/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "dict_builder"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+rayon = "1.8"
+dashmap = "5"
+serde_json = "1"

dict_builder/LICENSE

@@ -0,0 +1,62 @@
+---
+The data within this folder is primarily sourced from SCOWL: https://github.com/en-wl/wordlist
+---
+
+Copyright 2000-2019 by Kevin Atkinson
+
+  Permission to use, copy, modify, distribute and sell these word
+  lists, the associated scripts, the output created from the scripts,
+  and its documentation for any purpose is hereby granted without fee,
+  provided that the above copyright notice appears in all copies and
+  that both that copyright notice and this permission notice appear in
+  supporting documentation. Kevin Atkinson makes no representations
+  about the suitability of this array for any purpose. It is provided
+  "as is" without express or implied warranty.
+
+Copyright (c) J Ross Beresford 1993-1999. All Rights Reserved.
+
+  The following restriction is placed on the use of this publication:
+  if The UK Advanced Cryptics Dictionary is used in a software package
+  or redistributed in any form, the copyright notice must be
+  prominently displayed and the text of this document must be included
+  verbatim.
+
+  There are no other restrictions: I would like to see the list
+  distributed as widely as possible.
+
+Special credit also goes to Alan Beale <[email protected]> as he has
+given me an incredible amount of feedback and created a number of
+special lists (those found in the Supplement) in order to help improve
+the overall quality of SCOWL.
+
+Many sources were used in the creation of SCOWL, most of them were in
+the public domain or used indirectly.  For a full list please see the
+SCOWL readme.
+
+http://wordlist.aspell.net/
+
+---
+Additional data sourced from Wordnik: https://github.com/wordnik/wordlist
+---
+
+MIT License
+
+Copyright (c) 2020 Wordnik
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

dict_builder/README.md

@@ -0,0 +1,15 @@
+# Truncate Dictionary Utilities
+
+This crate builds the dictionaries, frequency lists and more that power Truncate.
+
+## Requirements
+
+The following files must be created:
+- `dict_builder/support_data/en_word_freqs.txt`
+  - Sourced from https://github.com/hermitdave/FrequencyWords/blob/master/content/2018/en/en_full.txt
+- `dict_builder/support_data/wordnik_wordlist.txt`
+  - Sourced from https://github.com/wordnik/wordlist
+- `dict_builder/support_data/objectionable.json`
+  - Generated from the `word_definitions` folder of this repo
+- `dict_builder/support_data/generated_scowl_wordlists/*`
+  - Containing the files from the `final` directory of the latest release of http://wordlist.aspell.net/