Week 20 of 2024
Development log of Otterhide
5 items
SQLite: Separate the populate logic to a function
On by
Also correct some context for some errors.
Currently the program exporting to SQLite always fails because some foreign key constraint is violated, but there is relatively little information to debug. In order to get more information I want to clean up the code a bit and maybe save a partially populated database so a failure can be further investigated.
index fa3fda0..7a9f818 100644
--- a/src/bin/otterhide-to-sqlite.rs
+++ b/src/bin/otterhide-to-sqlite.rs
@@ -58,6 +58,18 @@ async fn main() -> anyhow::Result<()> {
=
= let transaction = pool.begin().await?;
=
+ populate_database(simulation, pool)
+ .await
+ .context("Populating the database")?;
+
+ transaction.commit().await?;
+ connection.close().await.context("Closing the connection")
+}
+
+async fn populate_database(
+ simulation: History,
+ pool: sqlx::Pool<sqlx::Sqlite>,
+) -> Result<(), anyhow::Error> {
= for EventLogEntry { date, event } in simulation.events.iter() {
= match event {
= HistoricalEvent::ConstructionOrder(ConstructionOrder(building)) => {
@@ -105,7 +117,7 @@ async fn main() -> anyhow::Result<()> {
= .bind(person.birth_date.0.month() as u8)
= .execute(&pool)
= .await
- .context("Inserting a person")?;
+ .context(format!("Inserting a person who just immigrated"))?;
=
= sqlx::query(
= "Insert into migration (
@@ -117,6 +129,9 @@ async fn main() -> anyhow::Result<()> {
= .bind(person.id.to_string())
= .bind(date.year())
= .bind(date.month() as u8)
+ .execute(&pool)
+ .await
+ .context(format!("Inserting immigration data"))?;
= }
= HistoricalEvent::ChildIsBorn(ChildIsBorn { child }) => {
= sqlx::query(
@@ -166,7 +181,7 @@ async fn main() -> anyhow::Result<()> {
= .bind(date.month() as u8)
= .execute(&pool)
= .await
- .context("Inserting immigration data")?;
+ .context(format!("Updating due to a person moving in"))?;
= }
= HistoricalEvent::MovedOut(MovedOut { where_from, who }) => {
= sqlx::query(
@@ -176,7 +191,7 @@ async fn main() -> anyhow::Result<()> {
= where
= person = ?
= and building = ?
- and until_month is null"
+ and until_month is null",
= )
= .bind(date.year())
= .bind(date.month() as u8)
@@ -184,11 +199,9 @@ async fn main() -> anyhow::Result<()> {
= .bind(where_from.to_string())
= .execute(&pool)
= .await
- .context("Inserting immigration data")?;
+ .context(format!("Updating due to a person moving out"))?;
= }
= }
= }
-
- transaction.commit().await?;
- connection.close().await.context("Closing the connection")
+ Ok(())
=}SQLite: include processed entry to error context
On by
index 7a9f818..b5d5ab5 100644
--- a/src/bin/otterhide-to-sqlite.rs
+++ b/src/bin/otterhide-to-sqlite.rs
@@ -70,138 +70,148 @@ async fn populate_database(
= simulation: History,
= pool: sqlx::Pool<sqlx::Sqlite>,
=) -> Result<(), anyhow::Error> {
- for EventLogEntry { date, event } in simulation.events.iter() {
- match event {
- HistoricalEvent::ConstructionOrder(ConstructionOrder(building)) => {
- sqlx::query(
- "Insert into building (
- id ,
- address ,
- construction_year ,
- construction_month ,
- longitude ,
- latitude
- ) values ( ? , ? , ? , ? , ? , ? )",
- )
- .bind(building.id.to_string())
- .bind(building.address.to_string())
- .bind(date.year())
- .bind(date.month() as u8)
- .bind(building.transform.translation.x)
- .bind(building.transform.translation.z)
- .execute(&pool)
- .await
- .context("Inserting a building")?;
- }
- HistoricalEvent::NewDistrictEstablished(NewDistrictEstablished(district)) => {
- sqlx::query("Insert into district (name) values (?)")
- .bind(district.name.to_string())
- .execute(&pool)
- .await
- .context("Inserting a district")?;
- }
- HistoricalEvent::Immigrated(Immigrated(person)) => {
- sqlx::query(
- "Insert into person (
- id,
- name,
- sex,
- birth_year,
- birth_month
- ) values (?, ?, ?, ?, ?)",
- )
- .bind(person.id.to_string())
- .bind(person.name.to_string())
- .bind(person.sex.to_string())
- .bind(person.birth_date.0.year())
- .bind(person.birth_date.0.month() as u8)
- .execute(&pool)
- .await
- .context(format!("Inserting a person who just immigrated"))?;
-
- sqlx::query(
- "Insert into migration (
- person,
- arrival_year,
- arrival_month
- ) values (?, ?, ?)",
- )
- .bind(person.id.to_string())
- .bind(date.year())
- .bind(date.month() as u8)
- .execute(&pool)
- .await
- .context(format!("Inserting immigration data"))?;
- }
- HistoricalEvent::ChildIsBorn(ChildIsBorn { child }) => {
- sqlx::query(
- "Insert into person (
- id,
- name,
- sex,
- birth_year,
- birth_month
- ) values (?, ?, ?, ?, ?)",
- )
- .bind(child.id.to_string())
- .bind(child.name.to_string())
- .bind(child.sex.to_string())
- .bind(child.birth_date.0.year())
- .bind(child.birth_date.0.month() as u8)
- .execute(&pool)
- .await
- .context("Inserting a new born person data")?;
- }
- HistoricalEvent::Died(Died { person }) => {
- sqlx::query(
- "Update person set
- death_year = ?,
- death_month = ?
- where id = ?",
- )
- .bind(date.year())
- .bind(date.month() as u8)
- .bind(person.to_string())
- .execute(&pool)
- .await
- .context("Registering death")?;
- }
- HistoricalEvent::MovedIn(MovedIn { where_to, who }) => {
- sqlx::query(
- "Insert into residency (
- person,
- building,
- from_year,
- from_month
- ) values (?, ?, ?, ?)",
- )
- .bind(who.to_string())
- .bind(where_to.to_string())
- .bind(date.year())
- .bind(date.month() as u8)
- .execute(&pool)
- .await
- .context(format!("Updating due to a person moving in"))?;
- }
- HistoricalEvent::MovedOut(MovedOut { where_from, who }) => {
- sqlx::query(
- "Update residency set
- until_year = ?,
- until_month = ?
- where
- person = ?
- and building = ?
- and until_month is null",
- )
- .bind(date.year())
- .bind(date.month() as u8)
- .bind(who.to_string())
- .bind(where_from.to_string())
- .execute(&pool)
- .await
- .context(format!("Updating due to a person moving out"))?;
- }
- }
+ for entry in simulation.events.iter() {
+ process_log_entry(entry, &pool)
+ .await
+ .map_err(|error| error.context(format!("Processing {entry:#?}")))?;
= }
= Ok(())
=}
+
+async fn process_log_entry(
+ entry: &EventLogEntry,
+ pool: &sqlx::Pool<sqlx::Sqlite>,
+) -> Result<(), anyhow::Error> {
+ let EventLogEntry { date, event } = entry;
+ Ok(match event {
+ HistoricalEvent::ConstructionOrder(ConstructionOrder(building)) => {
+ sqlx::query(
+ "Insert into building (
+ id ,
+ address ,
+ construction_year ,
+ construction_month ,
+ longitude ,
+ latitude
+ ) values ( ? , ? , ? , ? , ? , ? )",
+ )
+ .bind(building.id.to_string())
+ .bind(building.address.to_string())
+ .bind(date.year())
+ .bind(date.month() as u8)
+ .bind(building.transform.translation.x)
+ .bind(building.transform.translation.z)
+ .execute(pool)
+ .await
+ .context("Inserting a building")?;
+ }
+ HistoricalEvent::NewDistrictEstablished(NewDistrictEstablished(district)) => {
+ sqlx::query("Insert into district (name) values (?)")
+ .bind(district.name.to_string())
+ .execute(pool)
+ .await
+ .context("Inserting a district")?;
+ }
+ HistoricalEvent::Immigrated(Immigrated(person)) => {
+ sqlx::query(
+ "Insert into person (
+ id,
+ name,
+ sex,
+ birth_year,
+ birth_month
+ ) values (?, ?, ?, ?, ?)",
+ )
+ .bind(person.id.to_string())
+ .bind(person.name.to_string())
+ .bind(person.sex.to_string())
+ .bind(person.birth_date.0.year())
+ .bind(person.birth_date.0.month() as u8)
+ .execute(pool)
+ .await
+ .context(format!("Inserting a person who just immigrated"))?;
+
+ sqlx::query(
+ "Insert into migration (
+ person,
+ arrival_year,
+ arrival_month
+ ) values (?, ?, ?)",
+ )
+ .bind(person.id.to_string())
+ .bind(date.year())
+ .bind(date.month() as u8)
+ .execute(pool)
+ .await
+ .context(format!("Inserting immigration data"))?;
+ }
+ HistoricalEvent::ChildIsBorn(ChildIsBorn { child }) => {
+ sqlx::query(
+ "Insert into person (
+ id,
+ name,
+ sex,
+ birth_year,
+ birth_month
+ ) values (?, ?, ?, ?, ?)",
+ )
+ .bind(child.id.to_string())
+ .bind(child.name.to_string())
+ .bind(child.sex.to_string())
+ .bind(child.birth_date.0.year())
+ .bind(child.birth_date.0.month() as u8)
+ .execute(pool)
+ .await
+ .context("Inserting a new born person data")?;
+ }
+ HistoricalEvent::Died(Died { person }) => {
+ sqlx::query(
+ "Update person set
+ death_year = ?,
+ death_month = ?
+ where id = ?",
+ )
+ .bind(date.year())
+ .bind(date.month() as u8)
+ .bind(person.to_string())
+ .execute(pool)
+ .await
+ .context("Registering death")?;
+ }
+ HistoricalEvent::MovedIn(MovedIn { where_to, who }) => {
+ sqlx::query(
+ "Insert into residency (
+ person,
+ building,
+ from_year,
+ from_month
+ ) values (?, ?, ?, ?)",
+ )
+ .bind(who.to_string())
+ .bind(where_to.to_string())
+ .bind(date.year())
+ .bind(date.month() as u8)
+ .execute(pool)
+ .await
+ .context(format!("Updating due to a person moving in"))?;
+ }
+ HistoricalEvent::MovedOut(MovedOut { where_from, who }) => {
+ sqlx::query(
+ "Update residency set
+ until_year = ?,
+ until_month = ?
+ where
+ person = ?
+ and building = ?
+ and until_month is null",
+ )
+ .bind(date.year())
+ .bind(date.month() as u8)
+ .bind(who.to_string())
+ .bind(where_from.to_string())
+ .execute(pool)
+ .await
+ .context(format!("Updating due to a person moving out"))?;
+ }
+ })
+}SQLite: Add an option to save failed database
On by
With the --failed-database option, if the populating process fails (as it currently always does), the otterhide-to-sqlite program will save a partially populated database at a given path. This should allow for further investigation as to the reason it failed.
index 63129cf..354bc3a 100644
--- a/Makefile
+++ b/Makefile
@@ -90,7 +90,7 @@ data/sqlite-export.db: check/sqlite-export/schema
=data/sqlite-export.db: src/**/*
=data/sqlite-export.db:
= mkdir --parents $(@D)
- cargo run --bin otterhide-to-sqlite $< $@
+ cargo run --bin otterhide-to-sqlite -- --failed-database=data/failed.db $< $@
=
=
=help: ## Print this help messageindex b5d5ab5..b3c3853 100644
--- a/src/bin/otterhide-to-sqlite.rs
+++ b/src/bin/otterhide-to-sqlite.rs
@@ -5,6 +5,7 @@ use otterhide::districts::NewDistrictEstablished;
=use otterhide::history::{EventLogEntry, HistoricalEvent, History};
=use otterhide::population::{ChildIsBorn, Died, Immigrated, MovedIn, MovedOut};
=use sqlx::sqlite::{SqliteConnectOptions, SqliteJournalMode, SqlitePool, SqliteSynchronous};
+use std::fs;
=use std::path::PathBuf;
=
=#[derive(Parser)]
@@ -14,6 +15,10 @@ struct Cli {
= simulation: PathBuf,
= /// Output .db file
= database: PathBuf,
+ /// A .db file to use in case of failure
+ #[arg(long)]
+ failed_database: Option<PathBuf>,
+
=}
=
=#[tokio::main(flavor = "current_thread")]
@@ -33,7 +38,7 @@ async fn main() -> anyhow::Result<()> {
= if db_filename.exists() {
= // TODO: Have a --force CLI argument and refuse to overwrite without it
= eprintln!("Overwriting {}", db_filename.display());
- std::fs::remove_file(&db_filename).context("Removing previous database")?;
+ std::fs::remove_file(&db_filename).context("Removing the previous database")?;
= }
=
= let connect_options = SqliteConnectOptions::new()
@@ -58,12 +63,22 @@ async fn main() -> anyhow::Result<()> {
=
= let transaction = pool.begin().await?;
=
- populate_database(simulation, pool)
+ let result = populate_database(simulation, pool)
= .await
- .context("Populating the database")?;
+ .context("Populating the database");
+
+ transaction.commit().await.context("Committing the transaction")?;
+ connection.close().await.context("Closing the database connection")?;
+
+ result.or_else(|error| {
+ eprintln!("Populating the database failed.");
+ if let Some(failed_db_filename) = cli.failed_database {
+ eprintln!("Saving the partially populated database at {}", failed_db_filename.display());
+ fs::copy(db_filename, failed_db_filename)?;
+ };
+ Err(error)
+ })
=
- transaction.commit().await?;
- connection.close().await.context("Closing the connection")
=}
=
=async fn populate_database(Add rlwrap to the environment
On by
For nicer SQLite interactions like history and line editing:
rlwrap sqlite3 data/sqlite-export.dbindex 203b60f..c0b2b26 100644
--- a/flake.nix
+++ b/flake.nix
@@ -42,12 +42,13 @@
= devShell = pkgs.mkShell {
= inherit buildInputs nativeBuildInputs;
= name = "${project-name}-develpoment-shell";
- packages = with pkgs; [
+ packages = [
= pkgs.rust-analyzer
= pkgs.miniserve
= pkgs.jq
= pkgs.watchexec
= pkgs.f3d
+ pkgs.rlwrap
= ];
= project_name = project-name; # Expose as an environment variable for make
= LD_LIBRARY_PATH = "$LD_LIBRARY_PATH:${ with pkgs; lib.makeLibraryPath buildInputs }";Fix: Child births not registered in history
On by
This was the reason for SQLite export failing, and probably few other bugs.
index 996d19c..162c4ce 100644
--- a/src/history.rs
+++ b/src/history.rs
@@ -147,6 +147,7 @@ fn register_historical_events(
= mut construction_orders: EventReader<ConstructionOrder>,
= mut new_districts: EventReader<NewDistrictEstablished>,
= mut immigrated: EventReader<Immigrated>,
+ mut born: EventReader<ChildIsBorn>,
= mut moved_in: EventReader<MovedIn>,
= mut moved_out: EventReader<MovedOut>,
= mut died: EventReader<Died>,
@@ -167,6 +168,11 @@ fn register_historical_events(
= debug!("Registering a historical event on {date:?}: {event:#?}");
= history.events.push(EventLogEntry { event, date });
= }
+ for event in born.read() {
+ let event = HistoricalEvent::ChildIsBorn(event.to_owned());
+ debug!("Registering a historical event on {date:?}: {event:#?}");
+ history.events.push(EventLogEntry { event, date });
+ }
= for event in moved_in.read() {
= let event = HistoricalEvent::MovedIn(event.to_owned());
= debug!("Registering a historical event on {date:?}: {event:#?}");
@@ -184,9 +190,6 @@ fn register_historical_events(
= }
=}
=
-// TODO: Use hankjordan/bevy_save with a custom pipeline
-// See https://github.com/hankjordan/bevy_save/blob/6fe1d55c822ecd075db9a86d383042c9fa07c192/examples/breakout.rs#L614
-
=/// The main snapshot that binds them all
=#[derive(Clone, Debug, Serialize, Deserialize)]
=pub struct MainSnapshot {