felixm/ledgerai
1
0
Fork 0
generated from felixm/defaultpy
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update project structure and move to beancount

Browse Source
This commit is contained in:
Felix Martin
2025-03-02 11:08:33 -05:00
parent 886bcdbdd1
commit 08c50e776e
17 changed files with 1844 additions and 296 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@@ -3,6 +3,8 @@
__pycache__/ __pycache__/
*.py[cod] *.py[cod]
*$py.class *$py.class
*.bean
*.beancount
# C extensions # C extensions
*.so *.so
@@ -159,4 +161,3 @@ cython_debug/
# and can be added to the global gitignore or merged into this file. For a more nuclear # and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder. # option (not recommended) you can uncomment the following to ignore the entire idea folder.
#.idea/ #.idea/

25
.pre-commit-config.yaml Normal file
View File

@@ -0,0 +1,25 @@
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v5.0.0
hooks:
- id: trailing-whitespace
- id: end-of-file-fixer
- id: check-yaml
- id: check-added-large-files
- id: check-ast
- id: check-json
- id: check-merge-conflict
- id: check-toml
- id: debug-statements
- id: detect-private-key
- id: mixed-line-ending
- repo: https://github.com/psf/black
rev: 25.1.0
hooks:
- id: black
- repo: https://github.com/pycqa/isort
rev: 6.0.1
hooks:
- id: isort

13
Pipfile
View File

@@ -1,13 +0,0 @@
[[source]]
url = "https://pypi.org/simple"
verify_ssl = true
name = "pypi"
[packages]
rich = "*"
pydantic = "*"
[dev-packages]
[requires]
python_version = "3.12"

164
Pipfile.lock generated
View File

@@ -1,164 +0,0 @@
{
"_meta": {
"hash": {
"sha256": "e15058e28401c3cc46c89647f6b7d652d432d4389072e5b18dd200b37c7af33d"
},
"pipfile-spec": 6,
"requires": {
"python_version": "3.12"
},
"sources": [
{
"name": "pypi",
"url": "https://pypi.org/simple",
"verify_ssl": true
}
]
},
"default": {
"annotated-types": {
"hashes": [
"sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53",
"sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"
],
"markers": "python_version >= '3.8'",
"version": "==0.7.0"
},
"markdown-it-py": {
"hashes": [
"sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1",
"sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb"
],
"markers": "python_version >= '3.8'",
"version": "==3.0.0"
},
"mdurl": {
"hashes": [
"sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8",
"sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"
],
"markers": "python_version >= '3.7'",
"version": "==0.1.2"
},
"pydantic": {
"hashes": [
"sha256:0c84efd9548d545f63ac0060c1e4d39bb9b14db8b3c0652338aecc07b5adec52",
"sha256:ee8538d41ccb9c0a9ad3e0e5f07bf15ed8015b481ced539a1759d8cc89ae90d0"
],
"index": "pypi",
"markers": "python_version >= '3.8'",
"version": "==2.7.4"
},
"pydantic-core": {
"hashes": [
"sha256:01dd777215e2aa86dfd664daed5957704b769e726626393438f9c87690ce78c3",
"sha256:0eb2a4f660fcd8e2b1c90ad566db2b98d7f3f4717c64fe0a83e0adb39766d5b8",
"sha256:0fbbdc827fe5e42e4d196c746b890b3d72876bdbf160b0eafe9f0334525119c8",
"sha256:123c3cec203e3f5ac7b000bd82235f1a3eced8665b63d18be751f115588fea30",
"sha256:14601cdb733d741b8958224030e2bfe21a4a881fb3dd6fbb21f071cabd48fa0a",
"sha256:18f469a3d2a2fdafe99296a87e8a4c37748b5080a26b806a707f25a902c040a8",
"sha256:19894b95aacfa98e7cb093cd7881a0c76f55731efad31073db4521e2b6ff5b7d",
"sha256:1b4de2e51bbcb61fdebd0ab86ef28062704f62c82bbf4addc4e37fa4b00b7cbc",
"sha256:1d886dc848e60cb7666f771e406acae54ab279b9f1e4143babc9c2258213daa2",
"sha256:1f4d26ceb5eb9eed4af91bebeae4b06c3fb28966ca3a8fb765208cf6b51102ab",
"sha256:21a5e440dbe315ab9825fcd459b8814bb92b27c974cbc23c3e8baa2b76890077",
"sha256:293afe532740370aba8c060882f7d26cfd00c94cae32fd2e212a3a6e3b7bc15e",
"sha256:2f5966897e5461f818e136b8451d0551a2e77259eb0f73a837027b47dc95dab9",
"sha256:2fd41f6eff4c20778d717af1cc50eca52f5afe7805ee530a4fbd0bae284f16e9",
"sha256:2fdf2156aa3d017fddf8aea5adfba9f777db1d6022d392b682d2a8329e087cef",
"sha256:3c40d4eaad41f78e3bbda31b89edc46a3f3dc6e171bf0ecf097ff7a0ffff7cb1",
"sha256:43d447dd2ae072a0065389092a231283f62d960030ecd27565672bd40746c507",
"sha256:44a688331d4a4e2129140a8118479443bd6f1905231138971372fcde37e43528",
"sha256:44c7486a4228413c317952e9d89598bcdfb06399735e49e0f8df643e1ccd0558",
"sha256:44cd83ab6a51da80fb5adbd9560e26018e2ac7826f9626bc06ca3dc074cd198b",
"sha256:46387e38bd641b3ee5ce247563b60c5ca098da9c56c75c157a05eaa0933ed154",
"sha256:4701b19f7e3a06ea655513f7938de6f108123bf7c86bbebb1196eb9bd35cf724",
"sha256:4748321b5078216070b151d5271ef3e7cc905ab170bbfd27d5c83ee3ec436695",
"sha256:4b06beb3b3f1479d32befd1f3079cc47b34fa2da62457cdf6c963393340b56e9",
"sha256:4d0dcc59664fcb8974b356fe0a18a672d6d7cf9f54746c05f43275fc48636851",
"sha256:4e99bc050fe65c450344421017f98298a97cefc18c53bb2f7b3531eb39bc7805",
"sha256:509daade3b8649f80d4e5ff21aa5673e4ebe58590b25fe42fac5f0f52c6f034a",
"sha256:51991a89639a912c17bef4b45c87bd83593aee0437d8102556af4885811d59f5",
"sha256:53db086f9f6ab2b4061958d9c276d1dbe3690e8dd727d6abf2321d6cce37fa94",
"sha256:564d7922e4b13a16b98772441879fcdcbe82ff50daa622d681dd682175ea918c",
"sha256:574d92eac874f7f4db0ca653514d823a0d22e2354359d0759e3f6a406db5d55d",
"sha256:578e24f761f3b425834f297b9935e1ce2e30f51400964ce4801002435a1b41ef",
"sha256:59ff3e89f4eaf14050c8022011862df275b552caef8082e37b542b066ce1ff26",
"sha256:5f09baa656c904807e832cf9cce799c6460c450c4ad80803517032da0cd062e2",
"sha256:6891a2ae0e8692679c07728819b6e2b822fb30ca7445f67bbf6509b25a96332c",
"sha256:6a750aec7bf431517a9fd78cb93c97b9b0c496090fee84a47a0d23668976b4b0",
"sha256:6f5c4d41b2771c730ea1c34e458e781b18cc668d194958e0112455fff4e402b2",
"sha256:77450e6d20016ec41f43ca4a6c63e9fdde03f0ae3fe90e7c27bdbeaece8b1ed4",
"sha256:81b5efb2f126454586d0f40c4d834010979cb80785173d1586df845a632e4e6d",
"sha256:823be1deb01793da05ecb0484d6c9e20baebb39bd42b5d72636ae9cf8350dbd2",
"sha256:834b5230b5dfc0c1ec37b2fda433b271cbbc0e507560b5d1588e2cc1148cf1ce",
"sha256:847a35c4d58721c5dc3dba599878ebbdfd96784f3fb8bb2c356e123bdcd73f34",
"sha256:86110d7e1907ab36691f80b33eb2da87d780f4739ae773e5fc83fb272f88825f",
"sha256:8951eee36c57cd128f779e641e21eb40bc5073eb28b2d23f33eb0ef14ffb3f5d",
"sha256:8a7164fe2005d03c64fd3b85649891cd4953a8de53107940bf272500ba8a788b",
"sha256:8b8bab4c97248095ae0c4455b5a1cd1cdd96e4e4769306ab19dda135ea4cdb07",
"sha256:90afc12421df2b1b4dcc975f814e21bc1754640d502a2fbcc6d41e77af5ec312",
"sha256:938cb21650855054dc54dfd9120a851c974f95450f00683399006aa6e8abb057",
"sha256:942ba11e7dfb66dc70f9ae66b33452f51ac7bb90676da39a7345e99ffb55402d",
"sha256:972658f4a72d02b8abfa2581d92d59f59897d2e9f7e708fdabe922f9087773af",
"sha256:97736815b9cc893b2b7f663628e63f436018b75f44854c8027040e05230eeddb",
"sha256:98906207f29bc2c459ff64fa007afd10a8c8ac080f7e4d5beff4c97086a3dabd",
"sha256:99457f184ad90235cfe8461c4d70ab7dd2680e28821c29eca00252ba90308c78",
"sha256:a0d829524aaefdebccb869eed855e2d04c21d2d7479b6cada7ace5448416597b",
"sha256:a2fdd81edd64342c85ac7cf2753ccae0b79bf2dfa063785503cb85a7d3593223",
"sha256:a55b5b16c839df1070bc113c1f7f94a0af4433fcfa1b41799ce7606e5c79ce0a",
"sha256:a642295cd0c8df1b86fc3dced1d067874c353a188dc8e0f744626d49e9aa51c4",
"sha256:ab86ce7c8f9bea87b9d12c7f0af71102acbf5ecbc66c17796cff45dae54ef9a5",
"sha256:abc267fa9837245cc28ea6929f19fa335f3dc330a35d2e45509b6566dc18be23",
"sha256:ae1d6df168efb88d7d522664693607b80b4080be6750c913eefb77e34c12c71a",
"sha256:b2ebef0e0b4454320274f5e83a41844c63438fdc874ea40a8b5b4ecb7693f1c4",
"sha256:b48ece5bde2e768197a2d0f6e925f9d7e3e826f0ad2271120f8144a9db18d5c8",
"sha256:b7cdf28938ac6b8b49ae5e92f2735056a7ba99c9b110a474473fd71185c1af5d",
"sha256:bb4462bd43c2460774914b8525f79b00f8f407c945d50881568f294c1d9b4443",
"sha256:bc4ff9805858bd54d1a20efff925ccd89c9d2e7cf4986144b30802bf78091c3e",
"sha256:c1322d7dd74713dcc157a2b7898a564ab091ca6c58302d5c7b4c07296e3fd00f",
"sha256:c67598100338d5d985db1b3d21f3619ef392e185e71b8d52bceacc4a7771ea7e",
"sha256:ca26a1e73c48cfc54c4a76ff78df3727b9d9f4ccc8dbee4ae3f73306a591676d",
"sha256:d323a01da91851a4f17bf592faf46149c9169d68430b3146dcba2bb5e5719abc",
"sha256:dc1803ac5c32ec324c5261c7209e8f8ce88e83254c4e1aebdc8b0a39f9ddb443",
"sha256:e00a3f196329e08e43d99b79b286d60ce46bed10f2280d25a1718399457e06be",
"sha256:e85637bc8fe81ddb73fda9e56bab24560bdddfa98aa64f87aaa4e4b6730c23d2",
"sha256:e858ac0a25074ba4bce653f9b5d0a85b7456eaddadc0ce82d3878c22489fa4ee",
"sha256:eae237477a873ab46e8dd748e515c72c0c804fb380fbe6c85533c7de51f23a8f",
"sha256:ebef0dd9bf9b812bf75bda96743f2a6c5734a02092ae7f721c048d156d5fabae",
"sha256:ec3beeada09ff865c344ff3bc2f427f5e6c26401cc6113d77e372c3fdac73864",
"sha256:f76d0ad001edd426b92233d45c746fd08f467d56100fd8f30e9ace4b005266e4",
"sha256:f85d05aa0918283cf29a30b547b4df2fbb56b45b135f9e35b6807cb28bc47951",
"sha256:f9899c94762343f2cc2fc64c13e7cae4c3cc65cdfc87dd810a31654c9b7358cc"
],
"markers": "python_version >= '3.8'",
"version": "==2.18.4"
},
"pygments": {
"hashes": [
"sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199",
"sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a"
],
"markers": "python_version >= '3.8'",
"version": "==2.18.0"
},
"rich": {
"hashes": [
"sha256:4edbae314f59eb482f54e9e30bf00d33350aaa94f4bfcd4e9e3110e64d0d7222",
"sha256:9be308cb1fe2f1f57d67ce99e95af38a1e2bc71ad9813b0e247cf7ffbcc3a432"
],
"index": "pypi",
"markers": "python_full_version >= '3.7.0'",
"version": "==13.7.1"
},
"typing-extensions": {
"hashes": [
"sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d",
"sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"
],
"markers": "python_version >= '3.8'",
"version": "==4.12.2"
}
},
"develop": {}
}

23
README.md
View File

@@ -1,17 +1,25 @@
# ledgerai # ledgerai
Script to transform CSV data into [ledger](https://ledger-cli.org/) accounting Script to transform CSV files into [beancount](https://beancount.github.io/docs/beancount_language_syntax.html) accounting files.
files.
# Usage ## Usage
Run `pipenv install -dev` to install all packages. To transform CSV data into Beancount run `toldg` via `python-poetry`.
Run `pipenv shell` to get venv shell. ```bash
poetry -P ${LEDGER_DATA_ROOT} run toldg
```
Run `pipenv install <package>` to install a package. To visualize the data with [fava](https://beancount.github.io/fava/) install all
dependencies via `python-poetry`, enable the venv and run `fava` from there.
# Architecture ```bash
poetry install
eval "$(poetry env activate)"
fava your_ledger.beancount
```
## Architecture
The script takes a directory in which it recursively searches for CSV and LDG The script takes a directory in which it recursively searches for CSV and LDG
files. From these files, it generates a single ledger accounting file that files. From these files, it generates a single ledger accounting file that
@@ -26,4 +34,3 @@ transactions can also get a more meaningful description and tags.
The mapping information are stored in a file `mappings.json`. It maps a unique The mapping information are stored in a file `mappings.json`. It maps a unique
identifier for each transaction (based on the filename and full CSV row) to a identifier for each transaction (based on the filename and full CSV row) to a
respective *account2*. respective *account2*.

1617
poetry.lock generated Normal file
View File

File diff suppressed because it is too large Load Diff

40
pyproject.toml Normal file
View File

@@ -0,0 +1,40 @@
[build-system]
requires = ["setuptools", "wheel"]
build-backend = "setuptools.build_meta"
[project]
name = "toldg"
version = "0.1.0"
description = "Tool to generate ledger files from csv"
readme = "README.md"
requires-python = ">=3.12,<4.0"
license = {text = "MIT"}
authors = [
{name = "Felix Martin", email = "mail@felixm.de"}
]
dependencies = [
"fava (>=1.30.1,<2.0.0)",
"pydantic (>=2.10.6,<3.0.0)",
"beancount (>=3.1.0,<4.0.0)",
"rich (>=13.9.4,<14.0.0)"
]
[tool.poetry.group.dev.dependencies]
pre-commit = "^4.1.0"
black = "^25.1.0"
isort = "^6.0.1"
pytest = "^8.3.4"
[project.scripts]
toldg = "toldg.__main__:main"
[tool.setuptools]
package-dir = {"" = "src"}
[tool.black]
line-length = 88
target-version = ["py312"]
[tool.isort]
profile = "black"
line_length = 88

4
src/__init__.py
View File

@@ -1,4 +0,0 @@
def hello():
print("Hello, seaman!")

1
src/toldg/__init__.py Normal file
View File

@@ -0,0 +1 @@
__version__ = "0.1.0"

16
toldg.py → src/toldg/__main__.py
View File

@@ -1,8 +1,10 @@
import logging import logging
import src.utils
import src.process
from rich.logging import RichHandler from rich.logging import RichHandler
from toldg.process import process_csv_files, process_ldg_files
from toldg.utils import load_config, remove_if_exists, write_meta
def init_logging(): def init_logging():
logging.basicConfig( logging.basicConfig(
@@ -15,11 +17,11 @@ def init_logging():
def main(): def main():
init_logging() init_logging()
config = src.utils.load_config() config = load_config()
src.utils.remove_if_exists(config.output_file) remove_if_exists(config.output_file)
src.utils.write_meta(config) write_meta(config)
src.process.process_ldg_files(config) process_ldg_files(config)
src.process.process_csv_files(config) process_csv_files(config)
if __name__ == "__main__": if __name__ == "__main__":

15
src/fzf.py → src/toldg/fzf.py
View File

@@ -2,19 +2,20 @@ import errno
import subprocess import subprocess
import sys import sys
EXECUTABLE_NAME = "fzf.exe" if sys.platform == "win32" else "fzf"
EXECUTABLE_NAME = 'fzf.exe' if sys.platform == 'win32' else 'fzf'
def iterfzf(iterable, prompt='> '): def iterfzf(iterable, prompt="> "):
cmd = [EXECUTABLE_NAME, '--prompt=' + prompt] cmd = [EXECUTABLE_NAME, "--prompt=" + prompt]
encoding = sys.getdefaultencoding() encoding = sys.getdefaultencoding()
proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=None) proc = subprocess.Popen(
cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=None
)
if proc.stdin is None: if proc.stdin is None:
return None return None
try: try:
lines = "\n".join(iterable) lines = "\n".join(iterable)
proc.stdin.write(lines.encode('utf-8')) proc.stdin.write(lines.encode("utf-8"))
proc.stdin.close() proc.stdin.close()
except IOError as e: except IOError as e:
if e.errno != errno.EPIPE and errno.EPIPE != 32: if e.errno != errno.EPIPE and errno.EPIPE != 32:
@@ -24,7 +25,7 @@ def iterfzf(iterable, prompt='> '):
if proc.stdout is None: if proc.stdout is None:
return None return None
decode = lambda t: t.decode(encoding) decode = lambda t: t.decode(encoding)
output = [decode(ln.strip(b'\r\n\0')) for ln in iter(proc.stdout.readline, b'')] output = [decode(ln.strip(b"\r\n\0")) for ln in iter(proc.stdout.readline, b"")]
try: try:
return output[0] return output[0]
except IndexError: except IndexError:

20
src/models.py → src/toldg/models.py
View File

@@ -1,10 +1,9 @@
from pydantic import BaseModel
from typing import List
from typing import Optional
from pathlib import Path from pathlib import Path
from typing import List, Optional
from pydantic import BaseModel
UNKNOWN_CATEGORY = 'account2' UNKNOWN_CATEGORY = "account2"
class CsvConfig(BaseModel): class CsvConfig(BaseModel):
@@ -13,8 +12,9 @@ class CsvConfig(BaseModel):
file_match_regex attribute to decide whether to apply a config for a file. file_match_regex attribute to decide whether to apply a config for a file.
If multiple configs match a single file we raise an exception. If multiple configs match a single file we raise an exception.
""" """
class Config: class Config:
extra = 'forbid' extra = "forbid"
account1: str account1: str
file_match_regex: str file_match_regex: str
@@ -23,8 +23,8 @@ class CsvConfig(BaseModel):
output_date_format: str = "%Y/%m/%d" output_date_format: str = "%Y/%m/%d"
skip: int = 1 skip: int = 1
delimiter: str = "," delimiter: str = ","
quotechar: str = "\"" quotechar: str = '"'
currency: str = "$" currency: str = "USD"
class Config(BaseModel): class Config(BaseModel):
@@ -43,8 +43,9 @@ class Config(BaseModel):
find_duplicates (bool): Flag to check and abort on duplicated transactions. Not find_duplicates (bool): Flag to check and abort on duplicated transactions. Not
really useful. really useful.
""" """
class Config: class Config:
extra = 'forbid' extra = "forbid"
input_directory: Path input_directory: Path
mappings_file: Path mappings_file: Path
@@ -60,8 +61,9 @@ class Transaction(BaseModel):
""" """
Class for ledger transaction to render into ldg file. Class for ledger transaction to render into ldg file.
""" """
class Config: class Config:
extra = 'forbid' extra = "forbid"
currency: str currency: str
debit: str debit: str

16
src/predict.py → src/toldg/predict.py
View File

@@ -1,19 +1,23 @@
from src.models import Transaction, UNKNOWN_CATEGORY
from src.fzf import iterfzf
from typing import List from typing import List
from toldg.fzf import iterfzf
from toldg.models import UNKNOWN_CATEGORY, Transaction
def get_sort_categories(): def get_sort_categories():
def sort_categories(row: str, categories: List[str]): def sort_categories(row: str, categories: List[str]):
if learn is None: if learn is None:
return return
_, _, probs = learn.predict(row) _, _, probs = learn.predict(row)
cat_to_prob = dict(zip(learn.dls.vocab[1],probs.tolist())) cat_to_prob = dict(zip(learn.dls.vocab[1], probs.tolist()))
categories.sort(key=lambda c: cat_to_prob[c] if c in cat_to_prob else 0.0, reverse=True) categories.sort(
key=lambda c: cat_to_prob[c] if c in cat_to_prob else 0.0, reverse=True
)
learn = None learn = None
try: try:
from fastai.text.all import load_learner from fastai.text.all import load_learner
learn = load_learner("export.pkl") learn = load_learner("export.pkl")
except ModuleNotFoundError: except ModuleNotFoundError:
user_input = input("No fastai module. Type yes to continue anyway.") user_input = input("No fastai module. Type yes to continue anyway.")
@@ -24,7 +28,9 @@ def get_sort_categories():
def add_account2(transactions: List[Transaction], categories: List[str]): def add_account2(transactions: List[Transaction], categories: List[str]):
unmapped_transactions = list(filter(lambda t: t.account2 == UNKNOWN_CATEGORY, transactions)) unmapped_transactions = list(
filter(lambda t: t.account2 == UNKNOWN_CATEGORY, transactions)
)
if len(unmapped_transactions) == 0: if len(unmapped_transactions) == 0:
return return
sort_categories = get_sort_categories() sort_categories = get_sort_categories()

70
src/process.py → src/toldg/process.py
View File

@@ -1,26 +1,26 @@
import csv import csv
import datetime
import logging import logging
import re import re
import sys import sys
import datetime from typing import Dict, List
import src.utils
import src.write import toldg.models
import src.models import toldg.predict
import src.predict import toldg.utils
from src.models import Config, CsvConfig, Transaction import toldg.write
from typing import List, Dict from toldg.models import Config, CsvConfig, Transaction
def process_ldg_files(config: Config): def process_ldg_files(config: Config):
for ldg_file in src.utils.get_ldg_files(config.input_directory): for ldg_file in toldg.utils.get_ldg_files(config.input_directory):
with open(ldg_file, 'r') as f_in: with open(ldg_file, "r") as f_in:
with open(config.output_file, 'a') as f_out: with open(config.output_file, "a") as f_out:
f_out.write(f_in.read()) f_out.write(f_in.read())
def get_csv_config(csv_file: str, csv_configs: List[CsvConfig]) -> CsvConfig: def get_csv_config(csv_file: str, csv_configs: List[CsvConfig]) -> CsvConfig:
cs = [c for c in csv_configs cs = [c for c in csv_configs if re.match(c.file_match_regex, csv_file)]
if re.match(c.file_match_regex, csv_file)]
if not cs: if not cs:
logging.critical(f"No CSV config for {csv_file}.") logging.critical(f"No CSV config for {csv_file}.")
sys.exit(1) sys.exit(1)
@@ -39,29 +39,28 @@ def get_transactions(csv_file: str, config: CsvConfig) -> List[Transaction]:
return amount[1:] if amount.startswith("-") else "-" + amount return amount[1:] if amount.startswith("-") else "-" + amount
def row_to_transaction(row, fields): def row_to_transaction(row, fields):
""" The user can configure the mapping of CSV fields to the three """The user can configure the mapping of CSV fields to the three
required fields date, amount and description via the CsvConfig. """ required fields date, amount and description via the CsvConfig."""
t = {field: row[index] for index, field in fields} t = {field: row[index] for index, field in fields}
amount = t['amount'] amount = t["amount"]
return Transaction( return Transaction(
currency=config.currency, currency=config.currency,
debit=flip_sign(amount), debit=flip_sign(amount),
credit=amount, credit=amount,
date=date_to_date(t['date']), date=date_to_date(t["date"]),
account1=config.account1, account1=config.account1,
account2=src.models.UNKNOWN_CATEGORY, account2=toldg.models.UNKNOWN_CATEGORY,
description=t['description'], description=t["description"],
csv_file=csv_file, csv_file=csv_file,
row=csv_file + ", " + ", ".join(row)) row=csv_file + ", " + ", ".join(row),
)
fields = [(i, f) for i, f in enumerate(config.fields) if f] fields = [(i, f) for i, f in enumerate(config.fields) if f]
with open(csv_file, 'r') as f: with open(csv_file, "r") as f:
reader = csv.reader(f, delimiter=config.delimiter, reader = csv.reader(f, delimiter=config.delimiter, quotechar=config.quotechar)
quotechar=config.quotechar)
for _ in range(config.skip): for _ in range(config.skip):
next(reader) next(reader)
transactions = [row_to_transaction(row, fields) transactions = [row_to_transaction(row, fields) for row in reader if row]
for row in reader if row]
return transactions return transactions
@@ -100,7 +99,7 @@ def apply_descriptions(transactions: List[Transaction], descriptions: Dict[str,
def process_csv_files(config: Config): def process_csv_files(config: Config):
csv_files = src.utils.get_csv_files(config.input_directory) csv_files = toldg.utils.get_csv_files(config.input_directory)
transactions = [] transactions = []
for csv_file in csv_files: for csv_file in csv_files:
csv_file = str(csv_file) csv_file = str(csv_file)
@@ -111,13 +110,12 @@ def process_csv_files(config: Config):
find_duplicates(transactions) find_duplicates(transactions)
if config.descriptions_file is not None: if config.descriptions_file is not None:
descriptions = src.utils.read_descriptions(config.descriptions_file) descriptions = toldg.utils.read_descriptions(config.descriptions_file)
apply_descriptions(transactions, descriptions) apply_descriptions(transactions, descriptions)
mappings = src.utils.read_mappings(config.mappings_file) mappings = toldg.utils.read_mappings(config.mappings_file)
apply_mappings(transactions, mappings) apply_mappings(transactions, mappings)
src.predict.add_account2(transactions, config.categories) toldg.predict.add_account2(transactions, config.categories)
src.utils.write_mappings(transactions, config.mappings_file) toldg.utils.write_mappings(transactions, config.mappings_file)
src.write.render_to_file(transactions, config) toldg.write.render_to_file(transactions, config)

62
src/utils.py → src/toldg/utils.py
View File

@@ -1,20 +1,23 @@
import json
import logging import logging
import os import os
import sys import sys
import logging
import json
from pathlib import Path from pathlib import Path
from typing import List, Dict from typing import Dict, List
from src.models import Config, Transaction
from pydantic import ValidationError from pydantic import ValidationError
from toldg.models import Config, Transaction
def get_files(directory: Path, ending="") -> List[Path]: def get_files(directory: Path, ending="") -> List[Path]:
""" Gets files from directory recursively in lexigraphic order. """ """Gets files from directory recursively in lexigraphic order."""
return [Path(os.path.join(subdir, f)) return [
for subdir, _, files in os.walk(directory) Path(os.path.join(subdir, f))
for f in files for subdir, _, files in os.walk(directory)
if f.endswith(ending)] for f in files
if f.endswith(ending)
]
def get_csv_files(directory: Path) -> List[Path]: def get_csv_files(directory: Path) -> List[Path]:
@@ -33,7 +36,7 @@ def load_config() -> Config:
sys.exit(1) sys.exit(1)
try: try:
with open(config_file, 'r') as f: with open(config_file, "r") as f:
config = Config(**json.load(f)) config = Config(**json.load(f))
except ValidationError as e: except ValidationError as e:
logging.critical(f"Could not validate {config_file}.") logging.critical(f"Could not validate {config_file}.")
@@ -45,15 +48,26 @@ def load_config() -> Config:
return config return config
def write_meta(config: Config): def category_to_bean(c: str) -> str:
with open(config.output_file, 'a') as f: sections = map(list, c.split(":"))
for category in config.categories: new_sections = []
f.write(f"account {category}\n") for section in sections:
f.write("\n") section[0] = section[0].upper()
new_sections.append("".join(section))
return ":".join(new_sections)
for commodity in config.commodities:
f.write(f"commodity {commodity}\n") def write_meta(config: Config):
with open(config.output_file, "a") as f:
for category in config.categories:
f.write(f"2017-01-01 open {category_to_bean(category)}\n")
f.write("\n") f.write("\n")
f.write('option "operating_currency" "USD"\n\n')
# Commodity section is not required for beancount
# for commodity in config.commodities:
# f.write(f"commodity {commodity}\n")
# f.write("\n")
def write_mappings(transactions: List[Transaction], mappings_file: Path): def write_mappings(transactions: List[Transaction], mappings_file: Path):
@@ -69,20 +83,20 @@ def write_mappings(transactions: List[Transaction], mappings_file: Path):
def read_mappings(mappings_file: Path) -> Dict[str, str]: def read_mappings(mappings_file: Path) -> Dict[str, str]:
with open(mappings_file, 'r') as f: with open(mappings_file, "r") as f:
account2_to_rows = json.load(f) account2_to_rows = json.load(f)
return {row: category return {
for category, rows in account2_to_rows.items() row: category for category, rows in account2_to_rows.items() for row in rows
for row in rows} }
def read_descriptions(descriptions_file: Path) -> Dict[str, str]: def read_descriptions(descriptions_file: Path) -> Dict[str, str]:
""" I am basic so the description file is currently a double row based """I am basic so the description file is currently a double row based
format where the first row matches the CSV row and the second one is the format where the first row matches the CSV row and the second one is the
description. """ description."""
descriptions = {} descriptions = {}
current_row = None current_row = None
with open(descriptions_file, 'r') as f: with open(descriptions_file, "r") as f:
for line in f.readlines(): for line in f.readlines():
if current_row is None: if current_row is None:
current_row = line.rstrip("\n") current_row = line.rstrip("\n")

32
src/toldg/write.py Normal file
View File

@@ -0,0 +1,32 @@
from pathlib import Path
from typing import List
from toldg.models import Config, Transaction
from toldg.utils import category_to_bean
BEANCOUNT_TRANSACTION_TEMPLATE = """
{t.date} * "{t.description}"
{t.account2:<40} {t.debit:<6} {t.currency}
{t.account1:<40} {t.credit:<6} {t.currency}
"""
def format(t):
t.date = t.date.replace("/", "-")
t.description = t.description.replace('"', '\\"')
if not t.debit.startswith("-"):
t.debit = " " + t.debit
if not t.credit.startswith("-"):
t.credit = " " + t.credit
t.account1 = category_to_bean(t.account1)
t.account2 = category_to_bean(t.account2)
if t.currency == "EUR":
t.debit = t.debit.replace(".", "|").replace(",", ".").replace("|", ",")
t.credit = t.credit.replace(".", "|").replace(",", ".").replace("|", ",")
return BEANCOUNT_TRANSACTION_TEMPLATE.format(t=t)
def render_to_file(transactions: List[Transaction], config: Config):
content = "".join(format(t) for t in transactions)
with open(config.output_file, "a") as f:
f.write(content)

17
src/write.py
View File

@@ -1,17 +0,0 @@
from pathlib import Path
from typing import List
from src.models import Transaction, Config
LEDGER_TRANSACTION_TEMPLATE = """
{t.date} {t.description} ; {t.row}
{t.account2} {t.currency} {t.debit}
{t.account1} {t.currency} {t.credit}
"""
def render_to_file(transactions: List[Transaction], config: Config):
content = "".join([LEDGER_TRANSACTION_TEMPLATE.format(t=t)
for t in transactions])
with open(config.output_file, 'a') as f:
f.write(content)