felixm/ledgerai
1
0
Fork 0
generated from felixm/defaultpy
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update project structure and move to beancount

Browse Source
This commit is contained in:
Felix Martin
2025-03-02 11:08:33 -05:00
parent 886bcdbdd1
commit 08c50e776e
17 changed files with 1844 additions and 296 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@@ -3,6 +3,8 @@
__pycache__/
*.py[cod]
*$py.class
*.bean
*.beancount
# C extensions
*.so
@@ -159,4 +161,3 @@ cython_debug/
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
#.idea/

25
.pre-commit-config.yaml Normal file
View File

@@ -0,0 +1,25 @@
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v5.0.0
hooks:
- id: trailing-whitespace
- id: end-of-file-fixer
- id: check-yaml
- id: check-added-large-files
- id: check-ast
- id: check-json
- id: check-merge-conflict
- id: check-toml
- id: debug-statements
- id: detect-private-key
- id: mixed-line-ending
- repo: https://github.com/psf/black
rev: 25.1.0
hooks:
- id: black
- repo: https://github.com/pycqa/isort
rev: 6.0.1
hooks:
- id: isort

13
Pipfile
View File

@@ -1,13 +0,0 @@
[[source]]
url = "https://pypi.org/simple"
verify_ssl = true
name = "pypi"
[packages]
rich = "*"
pydantic = "*"
[dev-packages]
[requires]
python_version = "3.12"

164
Pipfile.lock generated
View File

@@ -1,164 +0,0 @@
{
"_meta": {
"hash": {
"sha256": "e15058e28401c3cc46c89647f6b7d652d432d4389072e5b18dd200b37c7af33d"
},
"pipfile-spec": 6,
"requires": {
"python_version": "3.12"
},
"sources": [
{
"name": "pypi",
"url": "https://pypi.org/simple",
"verify_ssl": true
}
]
},
"default": {
"annotated-types": {
"hashes": [
"sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53",
"sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"
],
"markers": "python_version >= '3.8'",
"version": "==0.7.0"
},
"markdown-it-py": {
"hashes": [
"sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1",
"sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb"
],
"markers": "python_version >= '3.8'",
"version": "==3.0.0"
},
"mdurl": {
"hashes": [
"sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8",
"sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"
],
"markers": "python_version >= '3.7'",
"version": "==0.1.2"
},
"pydantic": {
"hashes": [
"sha256:0c84efd9548d545f63ac0060c1e4d39bb9b14db8b3c0652338aecc07b5adec52",
"sha256:ee8538d41ccb9c0a9ad3e0e5f07bf15ed8015b481ced539a1759d8cc89ae90d0"
],
"index": "pypi",
"markers": "python_version >= '3.8'",
"version": "==2.7.4"
},
"pydantic-core": {
"hashes": [
"sha256:01dd777215e2aa86dfd664daed5957704b769e726626393438f9c87690ce78c3",
"sha256:0eb2a4f660fcd8e2b1c90ad566db2b98d7f3f4717c64fe0a83e0adb39766d5b8",
"sha256:0fbbdc827fe5e42e4d196c746b890b3d72876bdbf160b0eafe9f0334525119c8",
"sha256:123c3cec203e3f5ac7b000bd82235f1a3eced8665b63d18be751f115588fea30",
"sha256:14601cdb733d741b8958224030e2bfe21a4a881fb3dd6fbb21f071cabd48fa0a",
"sha256:18f469a3d2a2fdafe99296a87e8a4c37748b5080a26b806a707f25a902c040a8",
"sha256:19894b95aacfa98e7cb093cd7881a0c76f55731efad31073db4521e2b6ff5b7d",
"sha256:1b4de2e51bbcb61fdebd0ab86ef28062704f62c82bbf4addc4e37fa4b00b7cbc",
"sha256:1d886dc848e60cb7666f771e406acae54ab279b9f1e4143babc9c2258213daa2",
"sha256:1f4d26ceb5eb9eed4af91bebeae4b06c3fb28966ca3a8fb765208cf6b51102ab",
"sha256:21a5e440dbe315ab9825fcd459b8814bb92b27c974cbc23c3e8baa2b76890077",
"sha256:293afe532740370aba8c060882f7d26cfd00c94cae32fd2e212a3a6e3b7bc15e",
"sha256:2f5966897e5461f818e136b8451d0551a2e77259eb0f73a837027b47dc95dab9",
"sha256:2fd41f6eff4c20778d717af1cc50eca52f5afe7805ee530a4fbd0bae284f16e9",
"sha256:2fdf2156aa3d017fddf8aea5adfba9f777db1d6022d392b682d2a8329e087cef",
"sha256:3c40d4eaad41f78e3bbda31b89edc46a3f3dc6e171bf0ecf097ff7a0ffff7cb1",
"sha256:43d447dd2ae072a0065389092a231283f62d960030ecd27565672bd40746c507",
"sha256:44a688331d4a4e2129140a8118479443bd6f1905231138971372fcde37e43528",
"sha256:44c7486a4228413c317952e9d89598bcdfb06399735e49e0f8df643e1ccd0558",
"sha256:44cd83ab6a51da80fb5adbd9560e26018e2ac7826f9626bc06ca3dc074cd198b",
"sha256:46387e38bd641b3ee5ce247563b60c5ca098da9c56c75c157a05eaa0933ed154",
"sha256:4701b19f7e3a06ea655513f7938de6f108123bf7c86bbebb1196eb9bd35cf724",
"sha256:4748321b5078216070b151d5271ef3e7cc905ab170bbfd27d5c83ee3ec436695",
"sha256:4b06beb3b3f1479d32befd1f3079cc47b34fa2da62457cdf6c963393340b56e9",
"sha256:4d0dcc59664fcb8974b356fe0a18a672d6d7cf9f54746c05f43275fc48636851",
"sha256:4e99bc050fe65c450344421017f98298a97cefc18c53bb2f7b3531eb39bc7805",
"sha256:509daade3b8649f80d4e5ff21aa5673e4ebe58590b25fe42fac5f0f52c6f034a",
"sha256:51991a89639a912c17bef4b45c87bd83593aee0437d8102556af4885811d59f5",
"sha256:53db086f9f6ab2b4061958d9c276d1dbe3690e8dd727d6abf2321d6cce37fa94",
"sha256:564d7922e4b13a16b98772441879fcdcbe82ff50daa622d681dd682175ea918c",
"sha256:574d92eac874f7f4db0ca653514d823a0d22e2354359d0759e3f6a406db5d55d",
"sha256:578e24f761f3b425834f297b9935e1ce2e30f51400964ce4801002435a1b41ef",
"sha256:59ff3e89f4eaf14050c8022011862df275b552caef8082e37b542b066ce1ff26",
"sha256:5f09baa656c904807e832cf9cce799c6460c450c4ad80803517032da0cd062e2",
"sha256:6891a2ae0e8692679c07728819b6e2b822fb30ca7445f67bbf6509b25a96332c",
"sha256:6a750aec7bf431517a9fd78cb93c97b9b0c496090fee84a47a0d23668976b4b0",
"sha256:6f5c4d41b2771c730ea1c34e458e781b18cc668d194958e0112455fff4e402b2",
"sha256:77450e6d20016ec41f43ca4a6c63e9fdde03f0ae3fe90e7c27bdbeaece8b1ed4",
"sha256:81b5efb2f126454586d0f40c4d834010979cb80785173d1586df845a632e4e6d",
"sha256:823be1deb01793da05ecb0484d6c9e20baebb39bd42b5d72636ae9cf8350dbd2",
"sha256:834b5230b5dfc0c1ec37b2fda433b271cbbc0e507560b5d1588e2cc1148cf1ce",
"sha256:847a35c4d58721c5dc3dba599878ebbdfd96784f3fb8bb2c356e123bdcd73f34",
"sha256:86110d7e1907ab36691f80b33eb2da87d780f4739ae773e5fc83fb272f88825f",
"sha256:8951eee36c57cd128f779e641e21eb40bc5073eb28b2d23f33eb0ef14ffb3f5d",
"sha256:8a7164fe2005d03c64fd3b85649891cd4953a8de53107940bf272500ba8a788b",
"sha256:8b8bab4c97248095ae0c4455b5a1cd1cdd96e4e4769306ab19dda135ea4cdb07",
"sha256:90afc12421df2b1b4dcc975f814e21bc1754640d502a2fbcc6d41e77af5ec312",
"sha256:938cb21650855054dc54dfd9120a851c974f95450f00683399006aa6e8abb057",
"sha256:942ba11e7dfb66dc70f9ae66b33452f51ac7bb90676da39a7345e99ffb55402d",
"sha256:972658f4a72d02b8abfa2581d92d59f59897d2e9f7e708fdabe922f9087773af",
"sha256:97736815b9cc893b2b7f663628e63f436018b75f44854c8027040e05230eeddb",
"sha256:98906207f29bc2c459ff64fa007afd10a8c8ac080f7e4d5beff4c97086a3dabd",
"sha256:99457f184ad90235cfe8461c4d70ab7dd2680e28821c29eca00252ba90308c78",
"sha256:a0d829524aaefdebccb869eed855e2d04c21d2d7479b6cada7ace5448416597b",
"sha256:a2fdd81edd64342c85ac7cf2753ccae0b79bf2dfa063785503cb85a7d3593223",
"sha256:a55b5b16c839df1070bc113c1f7f94a0af4433fcfa1b41799ce7606e5c79ce0a",
"sha256:a642295cd0c8df1b86fc3dced1d067874c353a188dc8e0f744626d49e9aa51c4",
"sha256:ab86ce7c8f9bea87b9d12c7f0af71102acbf5ecbc66c17796cff45dae54ef9a5",
"sha256:abc267fa9837245cc28ea6929f19fa335f3dc330a35d2e45509b6566dc18be23",
"sha256:ae1d6df168efb88d7d522664693607b80b4080be6750c913eefb77e34c12c71a",
"sha256:b2ebef0e0b4454320274f5e83a41844c63438fdc874ea40a8b5b4ecb7693f1c4",
"sha256:b48ece5bde2e768197a2d0f6e925f9d7e3e826f0ad2271120f8144a9db18d5c8",
"sha256:b7cdf28938ac6b8b49ae5e92f2735056a7ba99c9b110a474473fd71185c1af5d",
"sha256:bb4462bd43c2460774914b8525f79b00f8f407c945d50881568f294c1d9b4443",
"sha256:bc4ff9805858bd54d1a20efff925ccd89c9d2e7cf4986144b30802bf78091c3e",
"sha256:c1322d7dd74713dcc157a2b7898a564ab091ca6c58302d5c7b4c07296e3fd00f",
"sha256:c67598100338d5d985db1b3d21f3619ef392e185e71b8d52bceacc4a7771ea7e",
"sha256:ca26a1e73c48cfc54c4a76ff78df3727b9d9f4ccc8dbee4ae3f73306a591676d",
"sha256:d323a01da91851a4f17bf592faf46149c9169d68430b3146dcba2bb5e5719abc",
"sha256:dc1803ac5c32ec324c5261c7209e8f8ce88e83254c4e1aebdc8b0a39f9ddb443",
"sha256:e00a3f196329e08e43d99b79b286d60ce46bed10f2280d25a1718399457e06be",
"sha256:e85637bc8fe81ddb73fda9e56bab24560bdddfa98aa64f87aaa4e4b6730c23d2",
"sha256:e858ac0a25074ba4bce653f9b5d0a85b7456eaddadc0ce82d3878c22489fa4ee",
"sha256:eae237477a873ab46e8dd748e515c72c0c804fb380fbe6c85533c7de51f23a8f",
"sha256:ebef0dd9bf9b812bf75bda96743f2a6c5734a02092ae7f721c048d156d5fabae",
"sha256:ec3beeada09ff865c344ff3bc2f427f5e6c26401cc6113d77e372c3fdac73864",
"sha256:f76d0ad001edd426b92233d45c746fd08f467d56100fd8f30e9ace4b005266e4",
"sha256:f85d05aa0918283cf29a30b547b4df2fbb56b45b135f9e35b6807cb28bc47951",
"sha256:f9899c94762343f2cc2fc64c13e7cae4c3cc65cdfc87dd810a31654c9b7358cc"
],
"markers": "python_version >= '3.8'",
"version": "==2.18.4"
},
"pygments": {
"hashes": [
"sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199",
"sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a"
],
"markers": "python_version >= '3.8'",
"version": "==2.18.0"
},
"rich": {
"hashes": [
"sha256:4edbae314f59eb482f54e9e30bf00d33350aaa94f4bfcd4e9e3110e64d0d7222",
"sha256:9be308cb1fe2f1f57d67ce99e95af38a1e2bc71ad9813b0e247cf7ffbcc3a432"
],
"index": "pypi",
"markers": "python_full_version >= '3.7.0'",
"version": "==13.7.1"
},
"typing-extensions": {
"hashes": [
"sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d",
"sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"
],
"markers": "python_version >= '3.8'",
"version": "==4.12.2"
}
},
"develop": {}
}

23
README.md
View File

@@ -1,17 +1,25 @@
# ledgerai
Script to transform CSV data into [ledger](https://ledger-cli.org/) accounting
files.
Script to transform CSV files into [beancount](https://beancount.github.io/docs/beancount_language_syntax.html) accounting files.
# Usage
## Usage
Run `pipenv install -dev` to install all packages.
To transform CSV data into Beancount run `toldg` via `python-poetry`.
Run `pipenv shell` to get venv shell.
```bash
poetry -P ${LEDGER_DATA_ROOT} run toldg
```
Run `pipenv install <package>` to install a package.
To visualize the data with [fava](https://beancount.github.io/fava/) install all
dependencies via `python-poetry`, enable the venv and run `fava` from there.
# Architecture
```bash
poetry install
eval "$(poetry env activate)"
fava your_ledger.beancount
```
## Architecture
The script takes a directory in which it recursively searches for CSV and LDG
files. From these files, it generates a single ledger accounting file that
@@ -26,4 +34,3 @@ transactions can also get a more meaningful description and tags.
The mapping information are stored in a file `mappings.json`. It maps a unique
identifier for each transaction (based on the filename and full CSV row) to a
respective *account2*.

1617
poetry.lock generated Normal file
View File

File diff suppressed because it is too large Load Diff

40
pyproject.toml Normal file
View File

@@ -0,0 +1,40 @@
[build-system]
requires = ["setuptools", "wheel"]
build-backend = "setuptools.build_meta"
[project]
name = "toldg"
version = "0.1.0"
description = "Tool to generate ledger files from csv"
readme = "README.md"
requires-python = ">=3.12,<4.0"
license = {text = "MIT"}
authors = [
{name = "Felix Martin", email = "mail@felixm.de"}
]
dependencies = [
"fava (>=1.30.1,<2.0.0)",
"pydantic (>=2.10.6,<3.0.0)",
"beancount (>=3.1.0,<4.0.0)",
"rich (>=13.9.4,<14.0.0)"
]
[tool.poetry.group.dev.dependencies]
pre-commit = "^4.1.0"
black = "^25.1.0"
isort = "^6.0.1"
pytest = "^8.3.4"
[project.scripts]
toldg = "toldg.__main__:main"
[tool.setuptools]
package-dir = {"" = "src"}
[tool.black]
line-length = 88
target-version = ["py312"]
[tool.isort]
profile = "black"
line_length = 88

4
src/__init__.py
View File

@@ -1,4 +0,0 @@
def hello():
print("Hello, seaman!")

1
src/toldg/__init__.py Normal file
View File

@@ -0,0 +1 @@
__version__ = "0.1.0"

16
toldg.py → src/toldg/__main__.py
View File

@@ -1,8 +1,10 @@
import logging
import src.utils
import src.process
from rich.logging import RichHandler
from toldg.process import process_csv_files, process_ldg_files
from toldg.utils import load_config, remove_if_exists, write_meta
def init_logging():
logging.basicConfig(
@@ -15,11 +17,11 @@ def init_logging():
def main():
init_logging()
config = src.utils.load_config()
src.utils.remove_if_exists(config.output_file)
src.utils.write_meta(config)
src.process.process_ldg_files(config)
src.process.process_csv_files(config)
config = load_config()
remove_if_exists(config.output_file)
write_meta(config)
process_ldg_files(config)
process_csv_files(config)
if __name__ == "__main__":

15
src/fzf.py → src/toldg/fzf.py
View File

@@ -2,19 +2,20 @@ import errno
import subprocess
import sys
EXECUTABLE_NAME = 'fzf.exe' if sys.platform == 'win32' else 'fzf'
EXECUTABLE_NAME = "fzf.exe" if sys.platform == "win32" else "fzf"
def iterfzf(iterable, prompt='> '):
cmd = [EXECUTABLE_NAME, '--prompt=' + prompt]
def iterfzf(iterable, prompt="> "):
cmd = [EXECUTABLE_NAME, "--prompt=" + prompt]
encoding = sys.getdefaultencoding()
proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=None)
proc = subprocess.Popen(
cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=None
)
if proc.stdin is None:
return None
try:
lines = "\n".join(iterable)
proc.stdin.write(lines.encode('utf-8'))
proc.stdin.write(lines.encode("utf-8"))
proc.stdin.close()
except IOError as e:
if e.errno != errno.EPIPE and errno.EPIPE != 32:
@@ -24,7 +25,7 @@ def iterfzf(iterable, prompt='> '):
if proc.stdout is None:
return None
decode = lambda t: t.decode(encoding)
output = [decode(ln.strip(b'\r\n\0')) for ln in iter(proc.stdout.readline, b'')]
output = [decode(ln.strip(b"\r\n\0")) for ln in iter(proc.stdout.readline, b"")]
try:
return output[0]
except IndexError:

22
src/models.py → src/toldg/models.py
View File

@@ -1,10 +1,9 @@
from pydantic import BaseModel
from typing import List
from typing import Optional
from pathlib import Path
from typing import List, Optional
from pydantic import BaseModel
UNKNOWN_CATEGORY = 'account2'
UNKNOWN_CATEGORY = "account2"
class CsvConfig(BaseModel):
@@ -13,8 +12,9 @@ class CsvConfig(BaseModel):
file_match_regex attribute to decide whether to apply a config for a file.
If multiple configs match a single file we raise an exception.
"""
class Config:
extra = 'forbid'
extra = "forbid"
account1: str
file_match_regex: str
@@ -23,8 +23,8 @@ class CsvConfig(BaseModel):
output_date_format: str = "%Y/%m/%d"
skip: int = 1
delimiter: str = ","
quotechar: str = "\""
currency: str = "$"
quotechar: str = '"'
currency: str = "USD"
class Config(BaseModel):
@@ -39,12 +39,13 @@ class Config(BaseModel):
CSV files.
categories (List[str]): A list of account2s. An account has to be defined here
before it can be used in a mapping. Otherwise, ledger will complain.
commodities (List[str]): A list of commodities relevant to the data processing.
commodities (List[str]): A list of commodities relevant to the data processing.
find_duplicates (bool): Flag to check and abort on duplicated transactions. Not
really useful.
"""
class Config:
extra = 'forbid'
extra = "forbid"
input_directory: Path
mappings_file: Path
@@ -60,8 +61,9 @@ class Transaction(BaseModel):
"""
Class for ledger transaction to render into ldg file.
"""
class Config:
extra = 'forbid'
extra = "forbid"
currency: str
debit: str

16
src/predict.py → src/toldg/predict.py
View File

@@ -1,19 +1,23 @@
from src.models import Transaction, UNKNOWN_CATEGORY
from src.fzf import iterfzf
from typing import List
from toldg.fzf import iterfzf
from toldg.models import UNKNOWN_CATEGORY, Transaction
def get_sort_categories():
def sort_categories(row: str, categories: List[str]):
if learn is None:
return
_, _, probs = learn.predict(row)
cat_to_prob = dict(zip(learn.dls.vocab[1],probs.tolist()))
categories.sort(key=lambda c: cat_to_prob[c] if c in cat_to_prob else 0.0, reverse=True)
cat_to_prob = dict(zip(learn.dls.vocab[1], probs.tolist()))
categories.sort(
key=lambda c: cat_to_prob[c] if c in cat_to_prob else 0.0, reverse=True
)
learn = None
try:
from fastai.text.all import load_learner
learn = load_learner("export.pkl")
except ModuleNotFoundError:
user_input = input("No fastai module. Type yes to continue anyway.")
@@ -24,7 +28,9 @@ def get_sort_categories():
def add_account2(transactions: List[Transaction], categories: List[str]):
unmapped_transactions = list(filter(lambda t: t.account2 == UNKNOWN_CATEGORY, transactions))
unmapped_transactions = list(
filter(lambda t: t.account2 == UNKNOWN_CATEGORY, transactions)
)
if len(unmapped_transactions) == 0:
return
sort_categories = get_sort_categories()

70
src/process.py → src/toldg/process.py
View File

@@ -1,26 +1,26 @@
import csv
import datetime
import logging
import re
import sys
import datetime
import src.utils
import src.write
import src.models
import src.predict
from src.models import Config, CsvConfig, Transaction
from typing import List, Dict
from typing import Dict, List
import toldg.models
import toldg.predict
import toldg.utils
import toldg.write
from toldg.models import Config, CsvConfig, Transaction
def process_ldg_files(config: Config):
for ldg_file in src.utils.get_ldg_files(config.input_directory):
with open(ldg_file, 'r') as f_in:
with open(config.output_file, 'a') as f_out:
for ldg_file in toldg.utils.get_ldg_files(config.input_directory):
with open(ldg_file, "r") as f_in:
with open(config.output_file, "a") as f_out:
f_out.write(f_in.read())
def get_csv_config(csv_file: str, csv_configs: List[CsvConfig]) -> CsvConfig:
cs = [c for c in csv_configs
if re.match(c.file_match_regex, csv_file)]
cs = [c for c in csv_configs if re.match(c.file_match_regex, csv_file)]
if not cs:
logging.critical(f"No CSV config for {csv_file}.")
sys.exit(1)
@@ -39,29 +39,28 @@ def get_transactions(csv_file: str, config: CsvConfig) -> List[Transaction]:
return amount[1:] if amount.startswith("-") else "-" + amount
def row_to_transaction(row, fields):
""" The user can configure the mapping of CSV fields to the three
required fields date, amount and description via the CsvConfig. """
"""The user can configure the mapping of CSV fields to the three
required fields date, amount and description via the CsvConfig."""
t = {field: row[index] for index, field in fields}
amount = t['amount']
amount = t["amount"]
return Transaction(
currency=config.currency,
debit=flip_sign(amount),
credit=amount,
date=date_to_date(t['date']),
account1=config.account1,
account2=src.models.UNKNOWN_CATEGORY,
description=t['description'],
csv_file=csv_file,
row=csv_file + ", " + ", ".join(row))
currency=config.currency,
debit=flip_sign(amount),
credit=amount,
date=date_to_date(t["date"]),
account1=config.account1,
account2=toldg.models.UNKNOWN_CATEGORY,
description=t["description"],
csv_file=csv_file,
row=csv_file + ", " + ", ".join(row),
)
fields = [(i, f) for i, f in enumerate(config.fields) if f]
with open(csv_file, 'r') as f:
reader = csv.reader(f, delimiter=config.delimiter,
quotechar=config.quotechar)
with open(csv_file, "r") as f:
reader = csv.reader(f, delimiter=config.delimiter, quotechar=config.quotechar)
for _ in range(config.skip):
next(reader)
transactions = [row_to_transaction(row, fields)
for row in reader if row]
transactions = [row_to_transaction(row, fields) for row in reader if row]
return transactions
@@ -100,7 +99,7 @@ def apply_descriptions(transactions: List[Transaction], descriptions: Dict[str,
def process_csv_files(config: Config):
csv_files = src.utils.get_csv_files(config.input_directory)
csv_files = toldg.utils.get_csv_files(config.input_directory)
transactions = []
for csv_file in csv_files:
csv_file = str(csv_file)
@@ -111,13 +110,12 @@ def process_csv_files(config: Config):
find_duplicates(transactions)
if config.descriptions_file is not None:
descriptions = src.utils.read_descriptions(config.descriptions_file)
descriptions = toldg.utils.read_descriptions(config.descriptions_file)
apply_descriptions(transactions, descriptions)
mappings = src.utils.read_mappings(config.mappings_file)
mappings = toldg.utils.read_mappings(config.mappings_file)
apply_mappings(transactions, mappings)
src.predict.add_account2(transactions, config.categories)
src.utils.write_mappings(transactions, config.mappings_file)
src.write.render_to_file(transactions, config)
toldg.predict.add_account2(transactions, config.categories)
toldg.utils.write_mappings(transactions, config.mappings_file)
toldg.write.render_to_file(transactions, config)

62
src/utils.py → src/toldg/utils.py
View File

@@ -1,20 +1,23 @@
import json
import logging
import os
import sys
import logging
import json
from pathlib import Path
from typing import List, Dict
from src.models import Config, Transaction
from typing import Dict, List
from pydantic import ValidationError
from toldg.models import Config, Transaction
def get_files(directory: Path, ending="") -> List[Path]:
""" Gets files from directory recursively in lexigraphic order. """
return [Path(os.path.join(subdir, f))
for subdir, _, files in os.walk(directory)
for f in files
if f.endswith(ending)]
"""Gets files from directory recursively in lexigraphic order."""
return [
Path(os.path.join(subdir, f))
for subdir, _, files in os.walk(directory)
for f in files
if f.endswith(ending)
]
def get_csv_files(directory: Path) -> List[Path]:
@@ -33,7 +36,7 @@ def load_config() -> Config:
sys.exit(1)
try:
with open(config_file, 'r') as f:
with open(config_file, "r") as f:
config = Config(**json.load(f))
except ValidationError as e:
logging.critical(f"Could not validate {config_file}.")
@@ -45,15 +48,26 @@ def load_config() -> Config:
return config
def write_meta(config: Config):
with open(config.output_file, 'a') as f:
for category in config.categories:
f.write(f"account {category}\n")
f.write("\n")
def category_to_bean(c: str) -> str:
sections = map(list, c.split(":"))
new_sections = []
for section in sections:
section[0] = section[0].upper()
new_sections.append("".join(section))
return ":".join(new_sections)
for commodity in config.commodities:
f.write(f"commodity {commodity}\n")
def write_meta(config: Config):
with open(config.output_file, "a") as f:
for category in config.categories:
f.write(f"2017-01-01 open {category_to_bean(category)}\n")
f.write("\n")
f.write('option "operating_currency" "USD"\n\n')
# Commodity section is not required for beancount
# for commodity in config.commodities:
# f.write(f"commodity {commodity}\n")
# f.write("\n")
def write_mappings(transactions: List[Transaction], mappings_file: Path):
@@ -69,20 +83,20 @@ def write_mappings(transactions: List[Transaction], mappings_file: Path):
def read_mappings(mappings_file: Path) -> Dict[str, str]:
with open(mappings_file, 'r') as f:
with open(mappings_file, "r") as f:
account2_to_rows = json.load(f)
return {row: category
for category, rows in account2_to_rows.items()
for row in rows}
return {
row: category for category, rows in account2_to_rows.items() for row in rows
}
def read_descriptions(descriptions_file: Path) -> Dict[str, str]:
""" I am basic so the description file is currently a double row based
"""I am basic so the description file is currently a double row based
format where the first row matches the CSV row and the second one is the
description. """
description."""
descriptions = {}
current_row = None
with open(descriptions_file, 'r') as f:
with open(descriptions_file, "r") as f:
for line in f.readlines():
if current_row is None:
current_row = line.rstrip("\n")

32
src/toldg/write.py Normal file
View File

@@ -0,0 +1,32 @@
from pathlib import Path
from typing import List
from toldg.models import Config, Transaction
from toldg.utils import category_to_bean
BEANCOUNT_TRANSACTION_TEMPLATE = """
{t.date} * "{t.description}"
{t.account2:<40} {t.debit:<6} {t.currency}
{t.account1:<40} {t.credit:<6} {t.currency}
"""
def format(t):
t.date = t.date.replace("/", "-")
t.description = t.description.replace('"', '\\"')
if not t.debit.startswith("-"):
t.debit = " " + t.debit
if not t.credit.startswith("-"):
t.credit = " " + t.credit
t.account1 = category_to_bean(t.account1)
t.account2 = category_to_bean(t.account2)
if t.currency == "EUR":
t.debit = t.debit.replace(".", "|").replace(",", ".").replace("|", ",")
t.credit = t.credit.replace(".", "|").replace(",", ".").replace("|", ",")
return BEANCOUNT_TRANSACTION_TEMPLATE.format(t=t)
def render_to_file(transactions: List[Transaction], config: Config):
content = "".join(format(t) for t in transactions)
with open(config.output_file, "a") as f:
f.write(content)

17
src/write.py
View File

@@ -1,17 +0,0 @@
from pathlib import Path
from typing import List
from src.models import Transaction, Config
LEDGER_TRANSACTION_TEMPLATE = """
{t.date} {t.description} ; {t.row}
{t.account2} {t.currency} {t.debit}
{t.account1} {t.currency} {t.credit}
"""
def render_to_file(transactions: List[Transaction], config: Config):
content = "".join([LEDGER_TRANSACTION_TEMPLATE.format(t=t)
for t in transactions])
with open(config.output_file, 'a') as f:
f.write(content)