From d89e49e7c9632a87b2207cbea1689aa167388f76 Mon Sep 17 00:00:00 2001
From: Christoph Alt <christoph.alt@fau.de>
Date: Wed, 3 Aug 2022 16:41:59 +0200
Subject: [PATCH] convert all time stuff to ints
---
cbutil/data_points.py | 11 ++++++++++-
cbutil/postprocessing/plain_text.py | 7 ++++++-
cbutil/util.py | 15 +++++++++++++++
tests/test_csv.py | 2 ++
tests/test_util.py | 13 ++++++++++++-
5 files changed, 45 insertions(+), 3 deletions(-)
diff --git a/cbutil/data_points.py b/cbutil/data_points.py
index 3027f4e..ba1b436 100644
--- a/cbutil/data_points.py
+++ b/cbutil/data_points.py
@@ -1,4 +1,8 @@
from dataclasses import dataclass, asdict
+from .util import time_conversion
+import logging
+
+logger = logging.getLogger(__file__)
@dataclass
@@ -37,5 +41,10 @@ def data_point_factory(run, *,
tag_keys = run.keys() - no_tag_keys - field_keys - {time_key}
tags = {key: run[key]for key in tag_keys}
- time = run.get(time_key)
+ time = run[time_key]
+ try:
+ time = time_conversion(time)
+ except ValueError:
+ logger.warn(f"{time} could not be transformed with the current pattern")
+ pass
return DataPoint(measurement_name, time, fields=fields, tags=tags)
diff --git a/cbutil/postprocessing/plain_text.py b/cbutil/postprocessing/plain_text.py
index 9638aa1..501e8a2 100644
--- a/cbutil/postprocessing/plain_text.py
+++ b/cbutil/postprocessing/plain_text.py
@@ -1,5 +1,6 @@
from typing import Callable, Iterable
import csv
+from cbutil.util import file_time_to_sec
def process_linewise(func: Callable, lines: Iterable):
@@ -11,7 +12,11 @@ def process_linewise(func: Callable, lines: Iterable):
pass
-def iterate_csv(path):
+def iterate_csv(path, time_key="timestamp"):
+ file_time = file_time_to_sec(path)
+
with open(path, "r") as in_file:
for row in csv.DictReader(in_file):
+ if time_key not in row:
+ row[time_key] = file_time
yield row
diff --git a/cbutil/util.py b/cbutil/util.py
index 67cd852..bc2029a 100644
--- a/cbutil/util.py
+++ b/cbutil/util.py
@@ -1,4 +1,6 @@
import git
+import os
+from datetime import datetime
from typing import Union, Tuple
from pathlib import Path
@@ -44,3 +46,16 @@ def get_git_infos(repo_path: Union[Path, str], *, commit_key="commit", commit_ms
commit, commit_msg = get_commit_infos(get_current_head(get_repo(repo_path)))
commit_msg = remove_newline(commit_msg)
return {commit_key: commit, commit_msg_key: commit_msg}
+
+
+def file_time_to_sec(file_path) -> int:
+ return int(os.path.getmtime(file_path))
+
+
+def time_conversion(time_stamp, *, pattern="%Y-%m-%d %H:%M:%S"):
+ try:
+ return int(time_stamp)
+ except ValueError:
+ pass
+
+ return int(datetime.strptime(time_stamp, pattern).timestamp())
diff --git a/tests/test_csv.py b/tests/test_csv.py
index 5585c18..0105ab3 100644
--- a/tests/test_csv.py
+++ b/tests/test_csv.py
@@ -1,6 +1,8 @@
from cbutil.postprocessing.plain_text import iterate_csv
+from cbutil.util import file_time_to_sec
def test_iteration():
dicts = list(iterate_csv("tests/benchmark.csv"))
assert len(dicts) == 20
+ assert dicts[0]["timestamp"] == file_time_to_sec("tests/benchmark.csv")
diff --git a/tests/test_util.py b/tests/test_util.py
index aceb81e..d672955 100644
--- a/tests/test_util.py
+++ b/tests/test_util.py
@@ -1,8 +1,9 @@
-from cbutil.util import read_file_line_wise, get_git_infos
+from cbutil.util import read_file_line_wise, get_git_infos, time_conversion
from contextlib import contextmanager
from pathlib import Path
import os
import git
+import pytest
@contextmanager
@@ -28,3 +29,13 @@ def test_read_file():
def test_git_infos():
infos = get_git_infos(".", commit_key="commit", commit_msg_key="commit_message")
assert infos["commit"] == git.Repo(".").head.commit.hexsha
+
+
+def test_time_conv():
+ time_str = "2022-08-03 15:34:46"
+ assert time_conversion(time_str) == int(1659533686.0)
+ assert time_conversion(12341234) == 12341234
+ with pytest.raises(ValueError):
+ time_conversion("ajsdkfljaskldj")
+ with pytest.raises(ValueError):
+ time_conversion("2022-08-0315:34:46")
--
GitLab