From ab308de7d8a2d507d4c2aafd378be554dd34f360 Mon Sep 17 00:00:00 2001
From: safer-lgtm <git@code.fbi.h-da.de/safouan.erryfy>
Date: Mon, 27 May 2024 17:44:45 +0200
Subject: [PATCH] tests updated and docs created
---
docs/conf.py | 3 ++
docs/formulaone.rst | 61 +++++++++++++++++++++++++++++
docs/generated/dynamodb_helpers.rst | 19 ---------
docs/index.rst | 2 +-
docs/modules.rst | 7 ++++
formulaone/dynamodb_helpers.py | 55 +++++++++++++++++++++++++-
tests/test_dynamodb.py | 28 +++++++------
7 files changed, 140 insertions(+), 35 deletions(-)
create mode 100644 docs/formulaone.rst
delete mode 100644 docs/generated/dynamodb_helpers.rst
create mode 100644 docs/modules.rst
diff --git a/docs/conf.py b/docs/conf.py
index 97f4245..df7f067 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -32,6 +32,9 @@ extensions = [
'sphinx.ext.autosummary'
]
+#autosummary_generate = True # Enable autosummary
+
+
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
diff --git a/docs/formulaone.rst b/docs/formulaone.rst
new file mode 100644
index 0000000..35c40f6
--- /dev/null
+++ b/docs/formulaone.rst
@@ -0,0 +1,61 @@
+formulaone package
+==================
+
+Submodules
+----------
+
+formulaone.core module
+----------------------
+
+.. automodule:: formulaone.core
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+formulaone.dynamodb\_helpers module
+-----------------------------------
+
+.. automodule:: formulaone.dynamodb_helpers
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+formulaone.helpers module
+-------------------------
+
+.. automodule:: formulaone.helpers
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+formulaone.load\_latest\_race module
+------------------------------------
+
+.. automodule:: formulaone.load_latest_race
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+formulaone.prepare\_data module
+-------------------------------
+
+.. automodule:: formulaone.prepare_data
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+formulaone.tidy\_data module
+----------------------------
+
+.. automodule:: formulaone.tidy_data
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+Module contents
+---------------
+
+.. automodule:: formulaone
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/generated/dynamodb_helpers.rst b/docs/generated/dynamodb_helpers.rst
deleted file mode 100644
index 1926016..0000000
--- a/docs/generated/dynamodb_helpers.rst
+++ /dev/null
@@ -1,19 +0,0 @@
-sample.helpers
-==============
-
-.. automodule:: sample.helpers
- :members:
- :undoc-members:
- :show-inheritance:
-
- .. rubric:: Functions
-
- .. autosummary::
- :toctree: _autosummary
-
- get_raw_data_path
- get_dynamodb_resource
- list_dynamodb_tables
- get_movies_table
- get_movie_item
- query_movies_by_year
diff --git a/docs/index.rst b/docs/index.rst
index e9dd9c4..5b22772 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -9,6 +9,6 @@ Welcome to sample's documentation!
Contents:
.. toctree::
-
+ modules
usage
API
diff --git a/docs/modules.rst b/docs/modules.rst
new file mode 100644
index 0000000..b2a413d
--- /dev/null
+++ b/docs/modules.rst
@@ -0,0 +1,7 @@
+formulaone
+==========
+
+.. toctree::
+ :maxdepth: 4
+
+ formulaone
diff --git a/formulaone/dynamodb_helpers.py b/formulaone/dynamodb_helpers.py
index 822ebf2..dc18e09 100644
--- a/formulaone/dynamodb_helpers.py
+++ b/formulaone/dynamodb_helpers.py
@@ -1,6 +1,13 @@
-import boto3
+
+import os
+import sys
import configparser
+import boto3
+from botocore.exceptions import NoCredentialsError, PartialCredentialsError
from boto3.dynamodb.conditions import Key
+import pandas as pd
+from decimal import Decimal
+
def get_config():
"""
@@ -10,7 +17,17 @@ def get_config():
config: The config object.
"""
config = configparser.ConfigParser()
- config.read('config.ini')
+ #config_file = os.path.join(os.path.dirname(__file__), 'config.ini')
+ #config_file = os.path.abspath(os.path.join(os.getcwd(), 'config.ini'))
+ config_file = 'config.ini'
+ if not os.path.exists(config_file):
+ raise FileNotFoundError(f"The configuration file was not found: {config_file}")
+
+ config.read(config_file)
+
+ if 'AWS' not in config or 'DYNAMODB' not in config:
+ raise KeyError("One or more required sections are missing in the configuration file.")
+
return config
def get_dynamodb_resource():
@@ -21,6 +38,7 @@ def get_dynamodb_resource():
dynamo_resource: The DynamoDB resource.
"""
config = get_config()
+
aws_access_key_id = config['AWS']['aws_access_key_id']
aws_secret_access_key = config['AWS']['aws_secret_access_key']
region_name = config['AWS']['region_name']
@@ -34,6 +52,7 @@ def get_dynamodb_resource():
dynamo_resource = session.resource('dynamodb')
return dynamo_resource
+
def list_dynamodb_tables(dynamo_resource):
"""
Lists all tables in the DynamoDB resource.
@@ -91,3 +110,35 @@ def query_movies_by_year(table, year):
KeyConditionExpression=Key('year').eq(year)
)
return response.get('Items', [])
+
+
+def tidy_movie_data(movies):
+ """
+ Transforms the raw movie data into a tidy DataFrame.
+
+ Args:
+ movies (list): List of raw movie data.
+
+ Returns:
+ DataFrame: Tidy DataFrame containing movie data.
+ """
+ tidy_data = []
+ for movie in movies:
+ year = movie['year']
+ title = movie['title']
+ info = movie['info']
+ tidy_data.append({
+ 'year': int(year),
+ 'title': title,
+ 'actors': ', '.join(info['actors']),
+ 'release_date': info['release_date'],
+ 'plot': info['plot'],
+ 'genres': ', '.join(info['genres']),
+ 'image_url': info['image_url'],
+ 'directors': ', '.join(info['directors']),
+ 'rating': float(info['rating']),
+ 'rank': int(info['rank']),
+ 'running_time_secs': int(info['running_time_secs'])
+ })
+
+ return pd.DataFrame(tidy_data)
\ No newline at end of file
diff --git a/tests/test_dynamodb.py b/tests/test_dynamodb.py
index 28e6837..8d47739 100644
--- a/tests/test_dynamodb.py
+++ b/tests/test_dynamodb.py
@@ -6,20 +6,18 @@ import boto3
from botocore.exceptions import NoCredentialsError, PartialCredentialsError
# Add the formulaone model directory to the Python path
-sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+#sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
-from formulaone.dynamodb_helpers import (
- get_dynamodb_resource,
- list_dynamodb_tables,
- get_movies_table,
- get_movie_item,
- query_movies_by_year
-)
+module_path = os.path.abspath(os.path.join(os.getcwd(), 'formulaone'))
+sys.path.insert(0, module_path)
+print(sys.path.insert(0, module_path))
+
+from formulaone.dynamodb_helpers import *
def test_aws_config_keys():
"""Test the necessary AWS config keys."""
config = configparser.ConfigParser()
- config.read(os.path.join(os.path.dirname(__file__), '../config.ini'))
+ config.read(os.path.join(os.path.dirname(__file__), 'config.ini'))
assert 'AWS' in config, "AWS section is missing"
assert 'aws_access_key_id' in config['AWS'], "aws_access_key_id is missing in AWS"
assert 'aws_secret_access_key' in config['AWS'], "aws_secret_access_key is missing in AWS"
@@ -28,7 +26,7 @@ def test_aws_config_keys():
def test_dynamodb_config_keys():
"""Test the necessary DynamoDB config keys."""
config = configparser.ConfigParser()
- config.read(os.path.join(os.path.dirname(__file__), '../config.ini'))
+ config.read(os.path.join(os.path.dirname(__file__), 'config.ini'))
assert 'DYNAMODB' in config, "DYNAMODB section is missing"
assert 'table_name' in config['DYNAMODB'], "table_name is missing in DYNAMODB"
@@ -36,7 +34,7 @@ def test_dynamodb_connection():
"""Test the DynamoDB connection using the provided credentials and configuration."""
# Read the config file
config = configparser.ConfigParser()
- config.read(os.path.join(os.path.dirname(__file__), '../config.ini'))
+ config.read(os.path.join(os.path.dirname(__file__), 'config.ini'))
# Credentials and region
aws_access_key_id = config['AWS']['aws_access_key_id']
@@ -71,5 +69,9 @@ def test_dynamodb_resource():
dynamo_resource = get_dynamodb_resource()
assert dynamo_resource is not None, "DynamoDB resource should be initialized"
-if __name__ == "__main__":
- pytest.main()
+
+## Test ob die Tabelle da ist, noch da? Database da?
+# Soll in tidy format bringen
+# Doc erstellen lassen mit sphinx ( Aber zunächst soll die Methode gut kommentiert)
+# Zip oder url bei gitlab, Prof hinzufügen
+# log_transformation machen, testen ggf.
\ No newline at end of file
--
GitLab