diff --git a/cbutil/get_job_info.py b/cbutil/get_job_info.py index 093900371429443393f0b9e522accbea39c234b7..1a58d5ac17109406ef533ebc57ae5e04edd89aab 100644 --- a/cbutil/get_job_info.py +++ b/cbutil/get_job_info.py @@ -26,11 +26,18 @@ def get_api_url_pipelines(base_url: str, project_id: int, pipeline_id: int): def get_job_info(url: str): - jobs = requests.get(url) - if jobs.status_code != 200: - jobs.raise_for_status() - for job in jobs.json(): - yield job + next_url = url + while True: + logger.info(f"reqeuesting {next_url}") + jobs = requests.get(next_url) + if jobs.status_code != 200: + jobs.raise_for_status() + for job in jobs.json(): + yield job + if (next_page := jobs.headers['x-next-page']): + next_url = f"{url}?page={next_page}" + else: + break def create_job_datapoint(job: dict) -> DataPoint: diff --git a/cbutil/upload.py b/cbutil/upload.py index 189a68ab72f3f7ad6f20fe82405bf907971704ea..3718b5ae964bd7d554650abdefd2b6eb70066a4d 100644 --- a/cbutil/upload.py +++ b/cbutil/upload.py @@ -1,10 +1,11 @@ -import os import logging -import dotenv +import os import pprint -from influxdb import InfluxDBClient from dataclasses import dataclass +import dotenv +from influxdb import InfluxDBClient + logger = logging.getLogger(__file__) MISSING_DB_PW = """ @@ -52,14 +53,14 @@ class Uploader: database=config.database, ) - def upload(self, points, dry_run=False, *args, **kwargs): + def upload(self, points, dry_run=False, *, time_precision='s', **kwargs): logger.info(f"Uploading: {pprint.pformat(points)}") if (common_tags := kwargs.get("tags")): logger.info(f"with common tags: {pprint.pformat(common_tags)}") if not dry_run: success = self.client.write_points(points, - *args, + time_precision=time_precision, **kwargs) if success: logger.info(f"Uploaded {len(points)} items") diff --git a/cbutil/util.py b/cbutil/util.py index fa9ce1c3af9cad41abc26dffbdbb73366f573c5f..d42acf3995af5ed901bf9ebad591547848836ab6 100644 --- a/cbutil/util.py +++ b/cbutil/util.py @@ -56,7 +56,7 @@ def file_time_to_sec(file_path) -> int: return int(os.path.getmtime(file_path)) -def time_conversion(time_stamp, *, pattern="%Y-%m-%d %H:%M:%S%z"): +def time_conversion(time_stamp, *, pattern="%Y-%m-%d %H:%M:%S"): try: return int(time_stamp) except ValueError as e: