diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..5008ddf Binary files /dev/null and b/.DS_Store differ diff --git a/README.md b/README.md index f2f0a39..861d0ae 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,9 @@ This Python script automates the import of tasks (work packages) into a self-hosted OpenProject instance using the OpenProject REST API (v3). It reads a CSV file containing task details and posts them into the specified OpenProject project. Key features: -- Reads any user-provided CSV file containing `Subject`, `Description`, `Status`, and `Priority` columns. +- Reads any user-provided CSV file containing `Subject`, `Description`, `Status`, and `Priority` columns, as well as any custom columns specified in the first row of the CSV. - Authenticates using OpenProject API tokens (passed as Basic Auth with username `apikey`). -- Loops through all CSV rows and creates work packages (usually of type `Task`). +- Loops through all CSV rows and creates work packages (usually of type `Task`, although this can be customized using the Type column). - Provides basic error checking and API connectivity validation. ## Requirements @@ -21,11 +21,12 @@ pip install requests pandas ``` ## Script Functionality -- Prompts the user for: +- If a config is not specified, the script prompts the user for: - OpenProject URL (e.g., http://openproject.local) - OpenProject API token (generated from your user account page) - Numeric Project ID (e.g., 3) - Full path to the CSV file (e.g., D:/path/to/progress_import_ready.csv) +- if a config is provided as the first parameter, it should include the above mentioned params, look at the sample config provided. - Checks if the CSV file exists and loads it. - Validates the API token by making a test GET request. - Iterates over each row and posts a new work package via the API. @@ -38,6 +39,12 @@ Your CSV should include at least the following columns: - `Status` (e.g., "New", "In Progress", "Closed") - `Priority` (e.g., "Normal", "High", "Low") +## How to run the script +To run the script with a config file, make sure the config has the right values in it and type this into your terminal: +```bash +python import_requests_basicauth.py config.json +``` + ## How to Build an Executable To run this in an air-gapped environment as a standalone executable: @@ -63,7 +70,7 @@ dist/openproject_importer.exe openproject_importer.exe ``` -6️ Follow the prompts to enter your OpenProject connection details and import tasks. +6️ If you didn't use a config file, follow the prompts to enter your OpenProject connection details and import tasks. ## Notes - The script relies on the OpenProject REST API v3. diff --git a/config.json b/config.json new file mode 100644 index 0000000..af67af2 --- /dev/null +++ b/config.json @@ -0,0 +1,32 @@ +{ + "openproject_url": "OPEN_PROJECT_URL", + "api_token": "YOUR_PROJECT_API_TOKEN", + "project_id": PROJECT_ID, + "csv_file": "SAMPLE.csv", +"user_map": { + "custom_user": 1, + "custom_user2": 2 + }, + + "status_map": { + "New": 1, + "In Progress": 2, + "Closed": 3 + }, + + "priority_map": { + "Low": 7, + "Normal": 8, + "immediate": 10, + "High": 9 + }, + + "type_map": { + "Task": 1, + "Milestone":2, + "Summary task": 3 + } + + // add custom maps in here following the same format, make sure to use the correct name/numbers in your project + // remove this comment when executing script +} \ No newline at end of file diff --git a/import requests_basicauth.py b/import requests_basicauth.py deleted file mode 100644 index d6f994e..0000000 --- a/import requests_basicauth.py +++ /dev/null @@ -1,75 +0,0 @@ -import requests -import pandas as pd -import sys -import os -import base64 - -# Query user for configuration -openproject_url = input('Enter your OpenProject URL (e.g., https://openproject.local): ').strip().rstrip('/') -api_token = input('Enter your OpenProject API token: ').strip() -project_id = input('Enter your OpenProject numeric project ID (e.g., 5): ').strip() -csv_file = input('Enter the full path to the CSV file to import (e.g., D:/path/to/file.csv): ').strip() - -# Remove any surrounding quotes from the input path -csv_file = csv_file.strip('"').strip("'") - -# Check if file exists -if not os.path.isfile(csv_file): - print(f" The file '{csv_file}' does not exist. Please check the path and try again.") - sys.exit(1) - -# Load CSV -df = pd.read_csv(csv_file) - -# Set Basic Auth header using 'apikey' as username and API token as password -auth_header = base64.b64encode(f'apikey:{api_token}'.encode('utf-8')).decode('utf-8') -headers = { - 'Authorization': f'Basic {auth_header}', - 'Content-Type': 'application/json' -} - -# Test API connectivity -try: - test_response = requests.get(f"{openproject_url}/api/v3/projects/{project_id}", headers=headers) - if test_response.status_code != 200: - print(f" API token test failed — Status {test_response.status_code}: {test_response.text}") - sys.exit(1) - else: - print(f" API token verified. Proceeding with import.") -except Exception as e: - print(f" Failed to connect to API: {e}") - sys.exit(1) - -# Loop over each row and create work package -for idx, row in df.iterrows(): - payload = { - "_links": { - "project": { - "href": f"/api/v3/projects/{project_id}" - }, - "type": { - "href": f"/api/v3/types/1" # usually '1' is Task, adjust if needed - } - }, - "subject": row['Subject'], - "description": { - "format": "markdown", - "raw": row['Description'] - }, - "status": { - "name": row['Status'] - }, - "priority": { - "name": row['Priority'] - } - } - - response = requests.post(f"{openproject_url}/api/v3/work_packages", headers=headers, json=payload) - - if response.status_code == 201: - print(f" Created work package: {row['Subject']}") - else: - print(f" Failed to create {row['Subject']} — Status {response.status_code}: {response.text}") - -# Built by @SOCKS for OpenProject Ingestion -# https://github.com/773-process-312/OpenProject_CSV_Import.git diff --git a/import_requests_basicauth.py b/import_requests_basicauth.py new file mode 100644 index 0000000..438ae71 --- /dev/null +++ b/import_requests_basicauth.py @@ -0,0 +1,286 @@ +import requests +import pandas as pd +import sys +import os +import base64 +import math +import numpy as np +import json + +# this takes care of NaN errors, in case you don't have a value in your CSV +def scrub_nans(obj): + """Recursively replace NaN and infinity values with None so JSON encoding works.""" + if isinstance(obj, float): + if math.isnan(obj) or math.isinf(obj): + return None + return obj + if isinstance(obj, dict): + return {k: scrub_nans(v) for k, v in obj.items()} + if isinstance(obj, (list, tuple)): + return [scrub_nans(v) for v in obj] + return obj + + +def load_config(): + """Load config from JSON file if present, otherwise return empty dict.""" + # Allow optional config path, default to config.json + config_path = sys.argv[1] if len(sys.argv) > 1 else "config.json" + + if os.path.isfile(config_path): + try: + with open(config_path, "r", encoding="utf-8") as f: + cfg = json.load(f) + print(f"Loaded config from {config_path}") + return cfg + except Exception as e: + print(f"Could not read config file {config_path}: {e}") + return {} + else: + print(f"No config file found at {config_path}. Using interactive prompts.") + return {} + + +# Load config +config = load_config() + +# Helper for normalizing map keys +def _normalize_key(name): + if name is None: + return None + return str(name).lower().strip() + + +# Load maps from config and normalize keys to lowercase +raw_user_map = config.get("user_map", {}) +user_map = {_normalize_key(name): uid for name, uid in raw_user_map.items()} + +raw_status_map = config.get("status_map", {}) +status_map = {_normalize_key(name): sid for name, sid in raw_status_map.items()} + +raw_priority_map = config.get("priority_map", {}) +priority_map = {_normalize_key(name): pid for name, pid in raw_priority_map.items()} + +raw_type_map = config.get("type_map", {}) +type_map = {_normalize_key(name): tid for name, tid in raw_type_map.items()} + +raw_version_map = config.get("version_map", {}) +version_map = {_normalize_key(name): vid for name, vid in raw_version_map.items()} + +# Get settings from config or prompt the user +openproject_url = ( + config.get("openproject_url") + or input("Enter your OpenProject URL (for example, https://openproject.local): ") +).strip().rstrip("/") + +api_token = ( + config.get("api_token") + or input("Enter your OpenProject API token: ") +).strip() + +# Project id can be int in config, convert to string for URLs +project_id = config.get("project_id") +if project_id is None: + project_id = input("Enter your OpenProject numeric project ID (for example, 5): ").strip() +else: + project_id = str(project_id).strip() + +csv_file = ( + config.get("csv_file") + or config.get("csv_path") + or input("Enter the full path to the CSV file to import (for example, /path/to/file.csv): ") +).strip() + +# Remove any surrounding quotes from the input path +csv_file = csv_file.strip('"').strip("'") + +# Check if file exists +if not os.path.isfile(csv_file): + print(f"The file '{csv_file}' does not exist. Please check the path and try again.") + sys.exit(1) + +print(f"Using CSV file: {csv_file}") + +# Load CSV +df = pd.read_csv(csv_file, header=0) + +# Drop any anonymous "Unnamed" columns if present +df = df.loc[:, ~df.columns.str.startswith("Unnamed")] + +# Replace all NaN and NaT with None so JSON encoding is safe +df = df.replace({np.nan: None}) + +# Normalize column names: "Start Date" -> "start_date", "Assignee" -> "assignee" +col_map = {col: col.strip().lower().replace(" ", "_") for col in df.columns} +print("Detected columns:") +for orig, norm in col_map.items(): + print(f" {orig!r} -> {norm!r}") + +# Set Basic Auth header using "apikey" as username and API token as password +auth_header = base64.b64encode(f"apikey:{api_token}".encode("utf-8")).decode("utf-8") +headers = { + "Authorization": f"Basic {auth_header}", + "Content-Type": "application/json", +} + +# Test API connectivity +try: + test_response = requests.get( + f"{openproject_url}/api/v3/projects/{project_id}", headers=headers + ) + if test_response.status_code != 200: + print( + f"API token test failed - Status {test_response.status_code}: {test_response.text}" + ) + sys.exit(1) + else: + print("API token verified. Proceeding with import.") +except Exception as e: + print(f"Failed to connect to API: {e}") + sys.exit(1) + +# Track subjects seen in this run to avoid accidental duplicates inside one CSV +seen_subjects = set() + +# Loop over each row and create one work package per row +for idx, row in df.iterrows(): + # Normalize row into {normalized_column_name: value} + row_norm = {} + for orig_col, norm_col in col_map.items(): + row_norm[norm_col] = row.get(orig_col, None) + + # Subject is required + subject = row_norm.get("subject") + if not subject: + print(f"Row {idx}: missing 'subject', skipping") + continue + + if subject in seen_subjects: + print(f"Row {idx}: duplicate subject in CSV, skipping: {subject}") + continue + seen_subjects.add(subject) + + description_raw = row_norm.get("description") or "" + + # Base payload with required OpenProject bits + payload = { + "_links": { + "project": { + "href": f"/api/v3/projects/{project_id}" + }, + # type link may be overridden by type_map + "type": { + "href": "/api/v3/types/1" + }, + }, + "subject": subject, + } + + if description_raw: + payload["description"] = { + "format": "markdown", + "raw": description_raw, + } + + # Keys we do not send dynamically as plain strings because the API expects objects or links + reserved = { + "subject", + "description", + "status", + "priority", + "type", + "version", + "assignee", + "project", + } + + # Handle assignee using user_map (name -> user id) + assignee_name = row_norm.get("assignee") + if assignee_name: + lookup = _normalize_key(assignee_name) + user_id = user_map.get(lookup) + if user_id: + payload["_links"]["assignee"] = {"href": f"/api/v3/users/{user_id}"} + else: + print(f"Row {idx}: no user_map entry for assignee '{assignee_name}', leaving unassigned") + + # Handle status via status_map -> statuses/:id + status_name = row_norm.get("status") + if status_name: + lookup = _normalize_key(status_name) + status_id = status_map.get(lookup) + if status_id: + payload["_links"]["status"] = {"href": f"/api/v3/statuses/{status_id}"} + else: + print(f"Row {idx}: no status_map entry for status '{status_name}', leaving default") + + # Handle priority via priority_map -> priorities/:id + priority_name = row_norm.get("priority") + if priority_name: + lookup = _normalize_key(priority_name) + priority_id = priority_map.get(lookup) + if priority_id: + payload["_links"]["priority"] = {"href": f"/api/v3/priorities/{priority_id}"} + else: + print(f"Row {idx}: no priority_map entry for priority '{priority_name}', leaving default") + + # Handle type via type_map -> types/:id + type_name = row_norm.get("type") + if type_name: + lookup = _normalize_key(type_name) + type_id = type_map.get(lookup) + if type_id: + payload["_links"]["type"] = {"href": f"/api/v3/types/{type_id}"} + else: + print(f"Row {idx}: no type_map entry for type '{type_name}', keeping default /types/1") + + # Handle version via version_map -> versions/:id + version_name = row_norm.get("version") + if version_name: + lookup = _normalize_key(version_name) + version_id = version_map.get(lookup) + if version_id: + if "_links" not in payload: + payload["_links"] = {} + payload["_links"]["version"] = {"href": f"/api/v3/versions/{version_id}"} + else: + print(f"Row {idx}: no version_map entry for version '{version_name}', leaving unset") + + # Dynamically add everything else + for norm_key, value in row_norm.items(): + if norm_key in reserved: + continue + if value in (None, ""): + continue + + # Simple mapping for date-ish column names to OpenProject attribute names + if norm_key == "start_date": + payload["startDate"] = value + elif norm_key == "due_date": + payload["dueDate"] = value + elif norm_key == "closed_on": + payload["closedOn"] = value + else: + # Everything else goes in as is + payload[norm_key] = value + + clean_payload = scrub_nans(payload) + + # Print JSON for this task + print(f"\n=== Row {idx} subject: {subject} ===") + print(json.dumps(clean_payload, indent=2)) + + # Single POST per row + response = requests.post( + f"{openproject_url}/api/v3/work_packages", + headers=headers, + json=clean_payload, + ) + + if response.status_code == 201: + print(f"Created work package: {subject}") + else: + print( + f"Failed to create {subject} - Status {response.status_code}: {response.text}" + ) + +# Built by @SOCKS for OpenProject ingestion \ No newline at end of file