diff --git a/cbutil/data_points.py b/cbutil/data_points.py index 3027f4e2b99553c95b6b3f5b5eb947135c94b21c..ba1b436c54691144e18dade4ddb3681d39e39737 100644 --- a/cbutil/data_points.py +++ b/cbutil/data_points.py @@ -1,4 +1,8 @@ from dataclasses import dataclass, asdict +from .util import time_conversion +import logging + +logger = logging.getLogger(__file__) @dataclass @@ -37,5 +41,10 @@ def data_point_factory(run, *, tag_keys = run.keys() - no_tag_keys - field_keys - {time_key} tags = {key: run[key]for key in tag_keys} - time = run.get(time_key) + time = run[time_key] + try: + time = time_conversion(time) + except ValueError: + logger.warn(f"{time} could not be transformed with the current pattern") + pass return DataPoint(measurement_name, time, fields=fields, tags=tags) diff --git a/cbutil/postprocessing/plain_text.py b/cbutil/postprocessing/plain_text.py index 9638aa103e878d21736c2df20aec0f7ef654e1c7..501e8a284982511bdbffab72f011937463307200 100644 --- a/cbutil/postprocessing/plain_text.py +++ b/cbutil/postprocessing/plain_text.py @@ -1,5 +1,6 @@ from typing import Callable, Iterable import csv +from cbutil.util import file_time_to_sec def process_linewise(func: Callable, lines: Iterable): @@ -11,7 +12,11 @@ def process_linewise(func: Callable, lines: Iterable): pass -def iterate_csv(path): +def iterate_csv(path, time_key="timestamp"): + file_time = file_time_to_sec(path) + with open(path, "r") as in_file: for row in csv.DictReader(in_file): + if time_key not in row: + row[time_key] = file_time yield row diff --git a/cbutil/util.py b/cbutil/util.py index 67cd85285272cf3769dc6028d7ed810c759861a5..bc2029a6dd1d37429d102f85b565debdd1995cd2 100644 --- a/cbutil/util.py +++ b/cbutil/util.py @@ -1,4 +1,6 @@ import git +import os +from datetime import datetime from typing import Union, Tuple from pathlib import Path @@ -44,3 +46,16 @@ def get_git_infos(repo_path: Union[Path, str], *, commit_key="commit", commit_ms commit, commit_msg = get_commit_infos(get_current_head(get_repo(repo_path))) commit_msg = remove_newline(commit_msg) return {commit_key: commit, commit_msg_key: commit_msg} + + +def file_time_to_sec(file_path) -> int: + return int(os.path.getmtime(file_path)) + + +def time_conversion(time_stamp, *, pattern="%Y-%m-%d %H:%M:%S"): + try: + return int(time_stamp) + except ValueError: + pass + + return int(datetime.strptime(time_stamp, pattern).timestamp()) diff --git a/tests/test_csv.py b/tests/test_csv.py index 5585c185fec7b67198f08ce2d7e98cd4dee75523..0105ab37597dc5bb1c9cc766c114cce5a180f154 100644 --- a/tests/test_csv.py +++ b/tests/test_csv.py @@ -1,6 +1,8 @@ from cbutil.postprocessing.plain_text import iterate_csv +from cbutil.util import file_time_to_sec def test_iteration(): dicts = list(iterate_csv("tests/benchmark.csv")) assert len(dicts) == 20 + assert dicts[0]["timestamp"] == file_time_to_sec("tests/benchmark.csv") diff --git a/tests/test_util.py b/tests/test_util.py index aceb81e9b31f2675583573370550580a7de18d10..d672955a56687bdb741353fafe8d418fcba38c8a 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -1,8 +1,9 @@ -from cbutil.util import read_file_line_wise, get_git_infos +from cbutil.util import read_file_line_wise, get_git_infos, time_conversion from contextlib import contextmanager from pathlib import Path import os import git +import pytest @contextmanager @@ -28,3 +29,13 @@ def test_read_file(): def test_git_infos(): infos = get_git_infos(".", commit_key="commit", commit_msg_key="commit_message") assert infos["commit"] == git.Repo(".").head.commit.hexsha + + +def test_time_conv(): + time_str = "2022-08-03 15:34:46" + assert time_conversion(time_str) == int(1659533686.0) + assert time_conversion(12341234) == 12341234 + with pytest.raises(ValueError): + time_conversion("ajsdkfljaskldj") + with pytest.raises(ValueError): + time_conversion("2022-08-0315:34:46")