From 1fc5b12709f73cac8fef17edbbc721e2b78bd929 Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Tue, 30 Jul 2024 10:56:40 -0500 Subject: [PATCH 01/28] frontend project page --- frontend/src/actions/project.js | 103 ++++++++ frontend/src/app.config.ts | 2 + frontend/src/components/projects/Project.tsx | 241 ++++++++++++++++++ .../src/components/projects/ProjectCard.tsx | 121 +++++++++ .../components/projects/ProjectDetails.tsx | 32 +++ frontend/src/components/projects/Projects.tsx | 113 ++++++++ frontend/src/reducers/index.ts | 2 + frontend/src/reducers/project.ts | 29 +++ frontend/src/routes.tsx | 18 ++ 9 files changed, 661 insertions(+) create mode 100644 frontend/src/actions/project.js create mode 100644 frontend/src/components/projects/Project.tsx create mode 100644 frontend/src/components/projects/ProjectCard.tsx create mode 100644 frontend/src/components/projects/ProjectDetails.tsx create mode 100644 frontend/src/components/projects/Projects.tsx create mode 100644 frontend/src/reducers/project.ts diff --git a/frontend/src/actions/project.js b/frontend/src/actions/project.js new file mode 100644 index 000000000..c60b9eeb4 --- /dev/null +++ b/frontend/src/actions/project.js @@ -0,0 +1,103 @@ +// import { V2 } from "../openapi"; +// import { handleErrors } from "./common"; + +export const RECEIVE_PROJECTS = "RECEIVE_PROJECTS"; + +export function fetchProjects(skip = 0, limit = 12) { + return (dispatch) => { + dispatch({ + type: RECEIVE_PROJECTS, + projects: { + metadata: { + total_count: 3, + skip: skip, + limit: limit, + }, + data: [ + { + id: "60f9f8c8c23f5c45d8f0e0c2", + name: "Sample Project 1", + description: "A description of the first sample project", + created: "2024-07-29T12:00:00Z", + modified: "2024-07-29T12:00:00Z", + dataset_ids: [ + "669fcf3978f3222201e18a0d", + "669ea746d559628438e5788f", + ], + folder_ids: ["66a085640c20e43f5c50b059"], + file_ids: ["669ea735d559628438e57865"], + creator: { + id: "669ea726d559628438e57841", + first_name: "Chen", + last_name: "Wang", + email: "cwang138@illois.edu", + }, + }, + { + id: "60f9f8c8c23f5c45d8f0e0d1", + name: "Sample Project 2", + description: "A description of the second sample project", + created: "2024-07-28T12:00:00Z", + modified: "2024-07-28T12:00:00Z", + dataset_ids: ["669fcf3978f3222201e18a0d"], + folder_ids: [ + "66a085640c20e43f5c50b059", + "66a80284cf77abbb78b4435f", + ], + file_ids: ["669ea735d559628438e57865", "669ea733d559628438e57862"], + creator: { + id: "669ea726d559628438e57841", + first_name: "Chen", + last_name: "Wang", + email: "cwang138@illois.edu", + }, + }, + { + id: "60f9f8c8c23f5c45d8f0e0e2", + name: "Sample Project 3", + description: "A description of the third sample project", + created: "2024-07-27T12:00:00Z", + modified: "2024-07-27T12:00:00Z", + dataset_ids: [], + folder_ids: ["66a80284cf77abbb78b4435f"], + file_ids: [], + creator: { + id: "669ea726d559628438e57841", + first_name: "Chen", + last_name: "Wang", + email: "cwang138@illois.edu", + }, + }, + ], + }, + receivedAt: Date.now(), + }); + }; +} + +export const RECEIVE_PROJECT = "RECEIVE_PROJECT"; + +export function fetchProject(id) { + return (dispatch) => { + dispatch({ + type: RECEIVE_PROJECT, + project: { + id: "60f9f8c8c23f5c45d8f0e0c2", + name: "Sample Project", + description: "A description of the sample project", + created: "2024-07-29T12:00:00Z", + modified: "2024-07-29T12:00:00Z", + dataset_ids: ["60f9f8c8c23f5c45d8f0e0c3", "60f9f8c8c23f5c45d8f0e0c7"], + folder_ids: ["60f9f8c8c23f5c45d8f0e0c4"], + file_ids: ["60f9f8c8c23f5c45d8f0e0c5"], + creator: { + id: "60f9f8c8c23f5c45d8f0e0c6", + first_name: "Chen", + last_name: "Wang", + email: "cwang138@illois.edu", + }, + }, + receivedAt: Date.now(), + }); + }; +} diff --git a/frontend/src/app.config.ts b/frontend/src/app.config.ts index 7c325852c..de17e8a9f 100644 --- a/frontend/src/app.config.ts +++ b/frontend/src/app.config.ts @@ -25,6 +25,7 @@ interface Config { rawDataVisualizationThreshold: number; defaultFolderFilePerPage: number; defaultDatasetPerPage: number; + defaultProjectPerPage: number; defaultGroupPerPage: number; defaultUserPerPage: number; defaultApikeyPerPage: number; @@ -91,6 +92,7 @@ config["streamingBytes"] = 1024 * 10; // 10 MB? config["rawDataVisualizationThreshold"] = 1024 * 1024 * 10; // 10 MB config["defaultDatasetPerPage"] = 12; +config["defaultProjectPerPage"] = 12; config["defaultFolderFilePerPage"] = 5; config["defaultGroupPerPage"] = 5; config["defaultUserPerPage"] = 5; diff --git a/frontend/src/components/projects/Project.tsx b/frontend/src/components/projects/Project.tsx new file mode 100644 index 000000000..b2393fe32 --- /dev/null +++ b/frontend/src/components/projects/Project.tsx @@ -0,0 +1,241 @@ +import React, { ChangeEvent, useEffect, useState } from "react"; +import { + Box, + Grid, + Pagination, + Snackbar, + Tab, + Tabs, + Typography, +} from "@mui/material"; +import { useParams } from "react-router-dom"; +import { RootState } from "../../types/data"; +import { useDispatch, useSelector } from "react-redux"; +import { fetchProject } from "../../actions/project"; + +import { a11yProps, TabPanel } from "../tabs/TabComponent"; +// import FilesTable from "../files/FilesTable"; +import Layout from "../Layout"; +// import { ActionsMenuGroup } from "../datasets/ActionsMenuGroup"; +import { ProjectDetails } from "./ProjectDetails"; +import { FormatListBulleted, InsertDriveFile } from "@material-ui/icons"; +import AssessmentIcon from "@mui/icons-material/Assessment"; +import HistoryIcon from "@mui/icons-material/History"; +import ShareIcon from "@mui/icons-material/Share"; +import BuildIcon from "@mui/icons-material/Build"; +import { TabStyle } from "../../styles/Styles"; +import { ErrorModal } from "../errors/ErrorModal"; +import VisibilityIcon from "@mui/icons-material/Visibility"; +import config from "../../app.config"; + +export const Project = (): JSX.Element => { + // path parameter + const { projectId } = useParams<{ projectId?: string }>(); + + // Redux connect equivalent + const dispatch = useDispatch(); + const getProject = (projectId: string | null) => + dispatch(fetchProject(projectId)); + const project = useSelector((state: RootState) => state.project.project); + + // state + const [selectedTabIndex, setSelectedTabIndex] = useState(0); + + // Error msg dialog + const [errorOpen, setErrorOpen] = useState(false); + + // Snackbar + const [snackBarOpen, setSnackBarOpen] = useState(false); + const [snackBarMessage, setSnackBarMessage] = useState(""); + + const [paths, setPaths] = useState([]); + + const [currPageNum, setCurrPageNum] = useState(1); + const [limit] = useState(config.defaultFolderFilePerPage); + + useEffect(() => { + getProject(projectId); + }, [projectId]); + + // for breadcrumb + useEffect(() => { + // for breadcrumb + const tmpPaths = [ + { + name: project["name"], + url: `/projects/${projectId}`, + }, + ]; + + setPaths(tmpPaths); + }, [project]); + + const handleTabChange = ( + _event: React.ChangeEvent<{}>, + newTabIndex: number + ) => { + setSelectedTabIndex(newTabIndex); + }; + + const handlePageChange = (_: ChangeEvent, value: number) => { + const newSkip = (value - 1) * limit; + setCurrPageNum(value); + }; + + // @ts-ignore + return ( + + {/*Error Message dialogue*/} + + { + setSnackBarOpen(false); + setSnackBarMessage(""); + }} + message={snackBarMessage} + /> + + {/*title*/} + + + + + + {project["name"]} + + + {project["description"]} + + + + + + {/*actions*/} + + {/**/} + + {/*actions*/} + + + + + } + iconPosition="start" + sx={TabStyle} + label="Resources" + {...a11yProps(0)} + /> + } + iconPosition="start" + sx={TabStyle} + label="User Metadata" + {...a11yProps(1)} + disabled={false} + /> + } + iconPosition="start" + sx={TabStyle} + label="Machine Metadata" + {...a11yProps(2)} + disabled={false} + /> + } + iconPosition="start" + sx={TabStyle} + label="Analysis" + {...a11yProps(3)} + disabled={false} + /> + } + iconPosition="start" + sx={TabStyle} + label="Extraction History" + {...a11yProps(4)} + disabled={false} + /> + } + iconPosition="start" + sx={TabStyle} + label="Visualizations" + {...a11yProps(5)} + disabled={false} + /> + } + iconPosition="start" + sx={TabStyle} + label="Access Control" + {...a11yProps(6)} + disabled={false} + /> + + + {/**/} + + + + + + {/**/} + {/*)}*/} + + + {/**/} + + {/* Viewer is not allowed to submit to extractor*/} + + {/**/} + + + {/**/} + + + {/**/} + + + {/**/} + + + + + + + + ); +}; diff --git a/frontend/src/components/projects/ProjectCard.tsx b/frontend/src/components/projects/ProjectCard.tsx new file mode 100644 index 000000000..00295a1b7 --- /dev/null +++ b/frontend/src/components/projects/ProjectCard.tsx @@ -0,0 +1,121 @@ +import React from "react"; +import Card from "@mui/material/Card"; +import CardActions from "@mui/material/CardActions"; +import CardContent from "@mui/material/CardContent"; +import Typography from "@mui/material/Typography"; +import { Link } from "react-router-dom"; +import { parseDate } from "../../utils/common"; +import { + Box, + CardActionArea, + CardHeader, + IconButton, + Tooltip, +} from "@mui/material"; +import { + Download, + Folder, + InsertDriveFile, + Storage, +} from "@mui/icons-material"; +import config from "../../app.config"; + +type ProjectCardProps = { + id?: string; + name?: string; + author?: string; + created?: string | Date; + description?: string; + numFiles?: number; + numFolders?: number; + numDatasets?: number; +}; + +export default function ProjectCard(props: ProjectCardProps) { + const { + id, + name, + author, + created, + description, + numFiles, + numFolders, + numDatasets, + } = props; + + const formattedCreated = parseDate(created, "PP"); + const subheader = `${formattedCreated} \u00B7 ${author}`; + + return ( + + + + + + {description} + + + + + + + + + + {numFiles ?? 0} + + + + + + + + {numFolders ?? 0} + + + + + + + + {numDatasets ?? 0} + + + + + + + + + + + + ); +} diff --git a/frontend/src/components/projects/ProjectDetails.tsx b/frontend/src/components/projects/ProjectDetails.tsx new file mode 100644 index 000000000..d8afe1404 --- /dev/null +++ b/frontend/src/components/projects/ProjectDetails.tsx @@ -0,0 +1,32 @@ +import React from "react"; +import { Box, Typography } from "@mui/material"; +import { parseDate } from "../../utils/common"; +import { StackedList } from "../util/StackedList"; + +export function ProjectDetails(props) { + const { id, created, modified, creator } = props.details; + + const details = new Map< + string, + { value: string | undefined; info?: string } + >(); + details.set("Owner", { value: `${creator.first_name} ${creator.last_name}` }); + details.set("Created", { + value: parseDate(created), + info: "Date and time of project creation", + }); + details.set("Updated", { + value: parseDate(modified), + info: "Date and time of project modification", + }); + details.set("Project identifier", { value: id }); + + return ( + + + Details + + + + ); +} diff --git a/frontend/src/components/projects/Projects.tsx b/frontend/src/components/projects/Projects.tsx new file mode 100644 index 000000000..9d1a0ab5b --- /dev/null +++ b/frontend/src/components/projects/Projects.tsx @@ -0,0 +1,113 @@ +import React, { ChangeEvent, useEffect, useState } from "react"; +import { Box, Button, Grid, Pagination, Tab, Tabs } from "@mui/material"; + +import { useDispatch, useSelector } from "react-redux"; +import { fetchProjects } from "../../actions/project"; + +import ProjectCard from "./ProjectCard"; +import Layout from "../Layout"; +import { Link as RouterLink } from "react-router-dom"; +import { ErrorModal } from "../errors/ErrorModal"; +import config from "../../app.config"; +import { RootState } from "../../types/data"; + +export const Projects = (): JSX.Element => { + // Redux connect equivalent + const dispatch = useDispatch(); + const listProjects = (skip: number | undefined, limit: number | undefined) => + dispatch(fetchProjects(skip, limit)); + const projects = useSelector( + (state: RootState) => state.project.projects.data + ); + const pageMetadata = useSelector( + (state: RootState) => state.project.projects.metadata + ); + + const [currPageNum, setCurrPageNum] = useState(1); + const [limit] = useState(config.defaultProjectPerPage); + const [errorOpen, setErrorOpen] = useState(false); + + // Admin mode will fetch all projects + useEffect(() => { + listProjects((currPageNum - 1) * limit, limit); + }, [currPageNum, limit]); + + // pagination + const handlePageChange = (_: ChangeEvent, value: number) => { + const newSkip = (value - 1) * limit; + setCurrPageNum(value); + listProjects(newSkip, limit); + }; + + return ( + + {/*Error Message dialogue*/} + + + + + {projects !== undefined ? ( + projects.map((project) => { + return ( + + + + ); + }) + ) : ( + <> + )} + {projects.length === 0 ? ( + + +

+ Nobody has created any projects on this instance. Click + below to create a project! +

+ + + + ) : ( + <> + )} + + {projects.length !== 0 ? ( + + + + ) : ( + <> + )} + + + + ); +}; diff --git a/frontend/src/reducers/index.ts b/frontend/src/reducers/index.ts index c1543db60..f92f3b386 100644 --- a/frontend/src/reducers/index.ts +++ b/frontend/src/reducers/index.ts @@ -12,10 +12,12 @@ import group from "./group"; import visualization from "./visualization"; import publicVisualization from "./public_visualization"; import feeds from "./feeds"; +import project from "./project"; const rootReducer = combineReducers({ file: file, dataset: dataset, + project: project, publicDataset: publicDataset, publicFile: publicFile, folder: folder, diff --git a/frontend/src/reducers/project.ts b/frontend/src/reducers/project.ts new file mode 100644 index 000000000..8a0d491ed --- /dev/null +++ b/frontend/src/reducers/project.ts @@ -0,0 +1,29 @@ +import { RECEIVE_PROJECT, RECEIVE_PROJECTS } from "../actions/project"; +import { DataAction } from "../types/action"; +import { DatasetState } from "../types/data"; + +// @ts-ignore +const defaultState: DatasetState = { + projects: { + metadata: {}, + data: [], + }, + project: {}, +}; + +const dataset = (state = defaultState, action: DataAction) => { + switch (action.type) { + case RECEIVE_PROJECT: + return Object.assign({}, state, { + project: action.project, + }); + case RECEIVE_PROJECTS: + return Object.assign({}, state, { + projects: action.projects, + }); + default: + return state; + } +}; + +export default dataset; diff --git a/frontend/src/routes.tsx b/frontend/src/routes.tsx index cd9defe5b..fcb205bb3 100644 --- a/frontend/src/routes.tsx +++ b/frontend/src/routes.tsx @@ -45,6 +45,8 @@ import { MetadataDefinitionEntry } from "./components/metadata/MetadataDefinitio import { Feeds } from "./components/listeners/Feeds"; import { AllListeners } from "./components/listeners/AllListeners"; import { FeedEntry } from "./components/listeners/FeedEntry"; +import { Projects } from "./components/projects/Projects"; +import { Project } from "./components/projects/Project"; // https://dev.to/iamandrewluca/private-route-in-react-router-v6-lg5 const PrivateRoute = (props): JSX.Element => { @@ -130,6 +132,22 @@ export const AppRoutes = (): JSX.Element => { ) : ( } /> )} + + + + } + /> + + + + } + /> Date: Mon, 29 Jul 2024 13:36:12 -0500 Subject: [PATCH 02/28] initial commit new model --- backend/app/models/project.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 backend/app/models/project.py diff --git a/backend/app/models/project.py b/backend/app/models/project.py new file mode 100644 index 000000000..eab0ce347 --- /dev/null +++ b/backend/app/models/project.py @@ -0,0 +1,22 @@ +from datetime import datetime +from enum import Enum, auto +from typing import List, Optional + +import pymongo +from app.models.authorization import AuthorizationDB, RoleType +from app.models.groups import GroupOut +from app.models.users import UserOut +from beanie import Document, PydanticObjectId, View +from pydantic import BaseModel, Field + + +class Project(BaseModel): + id: PydanticObjectId = Field(default_factory=PydanticObjectId, alias="_id") + name: str + description: Optional[str] = None + created: datetime = Field(default_factory=datetime.utcnow) + modified: datetime = Field(default_factory=datetime.utcnow) + dataset_ids: Optional[List[PydanticObjectId]] = None + folder_ids: Optional[List[PydanticObjectId]] = None + file_ids: Optional[List[PydanticObjectId]] = None + creator: UserOut \ No newline at end of file From 6540704c5fd79ee2a8fca979e0a45d0eb07a8c0a Mon Sep 17 00:00:00 2001 From: toddn Date: Mon, 29 Jul 2024 13:38:48 -0500 Subject: [PATCH 03/28] adding db, out, in, etc --- backend/app/models/project.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/backend/app/models/project.py b/backend/app/models/project.py index eab0ce347..76bb27bb3 100644 --- a/backend/app/models/project.py +++ b/backend/app/models/project.py @@ -10,7 +10,7 @@ from pydantic import BaseModel, Field -class Project(BaseModel): +class ProjectBase(BaseModel): id: PydanticObjectId = Field(default_factory=PydanticObjectId, alias="_id") name: str description: Optional[str] = None @@ -19,4 +19,15 @@ class Project(BaseModel): dataset_ids: Optional[List[PydanticObjectId]] = None folder_ids: Optional[List[PydanticObjectId]] = None file_ids: Optional[List[PydanticObjectId]] = None - creator: UserOut \ No newline at end of file + creator: UserOut + +class ProjectDB(Document, ProjectBase): + class Settings: + name = "projects" + +class ProjectIn(ProjectBase): + pass + +class ProjectOut(ProjectDB): + class Config: + fields = {"id": "id"} \ No newline at end of file From a5e6ea0ad2ca7e225e99f72c6fe41daccd7a3283 Mon Sep 17 00:00:00 2001 From: toddn Date: Mon, 29 Jul 2024 13:46:45 -0500 Subject: [PATCH 04/28] save and get methods --- backend/app/routers/projects.py | 83 +++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 backend/app/routers/projects.py diff --git a/backend/app/routers/projects.py b/backend/app/routers/projects.py new file mode 100644 index 000000000..f9c1fd086 --- /dev/null +++ b/backend/app/routers/projects.py @@ -0,0 +1,83 @@ +import datetime +import hashlib +import io +import os +import shutil +import tempfile +import zipfile +from collections.abc import Iterable, Mapping +from typing import List, Optional +from app.models.pages import Paged, _construct_page_metadata, _get_page_query + + +from app import dependencies +from app.config import settings + +from app.models.project import ( + ProjectBase, + ProjectDB, + ProjectIn, + ProjectOut, +) +from app.keycloak_auth import get_current_user, get_token, get_user +from beanie import PydanticObjectId +from beanie.operators import And, Or +from bson import ObjectId, json_util +from elasticsearch import Elasticsearch +from fastapi import APIRouter, Depends, File, HTTPException, Request, UploadFile +from fastapi.responses import StreamingResponse +from fastapi.security import HTTPBearer +from minio import Minio +from pika.adapters.blocking_connection import BlockingChannel +from pymongo import DESCENDING +from rocrate.model.person import Person +from rocrate.rocrate import ROCrate + +router = APIRouter() +security = HTTPBearer() + +clowder_bucket = os.getenv("MINIO_BUCKET_NAME", "clowder") + +@router.post("", response_model=ProjectOut) +async def save_project( + project_in: ProjectIn, + license_id: str, + user=Depends(get_current_user), + es: Elasticsearch = Depends(dependencies.get_elasticsearchclient), +): + + project = ProjectDB( + **project_in.dict(), + creator=user, + ) + await project.insert() + + # TODO Add new entry to elasticsearch + return project.dict() + +@router.get("", response_model=Paged) +async def get_projects( + user_id=Depends(get_user), + skip: int = 0, + limit: int = 10, + mine: bool = False, + enable_admin: bool = False, +): + + query = (ProjectDB.creator.email == user_id) + + projects_and_count = await ProjectDB.find(*query).to_list() + + page_metadata = _construct_page_metadata(projects_and_count, skip, limit) + # TODO have to change _id this way otherwise it won't work + # TODO need to research if there is other pydantic trick to make it work + + page = Paged( + metadata=page_metadata, + data=[ + ProjectOut(id=item.pop("_id"), **item) + for item in projects_and_count[0]["data"] + ], + ) + + return page.dict() \ No newline at end of file From 3023c623254e6d019e19ce2d8d1a4dd5d2e90ab5 Mon Sep 17 00:00:00 2001 From: toddn Date: Mon, 29 Jul 2024 13:53:13 -0500 Subject: [PATCH 05/28] get method --- backend/app/routers/projects.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/backend/app/routers/projects.py b/backend/app/routers/projects.py index f9c1fd086..8ff50ca21 100644 --- a/backend/app/routers/projects.py +++ b/backend/app/routers/projects.py @@ -80,4 +80,13 @@ async def get_projects( ], ) - return page.dict() \ No newline at end of file + return page.dict() + +@router.get("/{project_id}", response_model=ProjectOut) +async def get_project( + project_id: str, +): + if project := await ProjectDB.find_one(ProjectDB.id == PydanticObjectId(project_id) + ) is not None: + return project.dict() + raise HTTPException(status_code=404, detail=f"Project {project_id} not found") \ No newline at end of file From e71fc409c2a8bebe772e63d6726a50de4f77dcb7 Mon Sep 17 00:00:00 2001 From: toddn Date: Mon, 29 Jul 2024 15:45:13 -0500 Subject: [PATCH 06/28] add users to project --- backend/app/models/project.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/backend/app/models/project.py b/backend/app/models/project.py index 76bb27bb3..adaa9a7ff 100644 --- a/backend/app/models/project.py +++ b/backend/app/models/project.py @@ -10,6 +10,10 @@ from pydantic import BaseModel, Field +class Member(BaseModel): + user: UserOut + editor: bool = False + class ProjectBase(BaseModel): id: PydanticObjectId = Field(default_factory=PydanticObjectId, alias="_id") name: str @@ -20,6 +24,7 @@ class ProjectBase(BaseModel): folder_ids: Optional[List[PydanticObjectId]] = None file_ids: Optional[List[PydanticObjectId]] = None creator: UserOut + users: List[Member] = [] class ProjectDB(Document, ProjectBase): class Settings: From e030b8723e07a9f248a42ba4e5a1a1c7db9ec376 Mon Sep 17 00:00:00 2001 From: toddn Date: Mon, 29 Jul 2024 15:46:13 -0500 Subject: [PATCH 07/28] adding to do --- backend/app/routers/projects.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/app/routers/projects.py b/backend/app/routers/projects.py index 8ff50ca21..2d8c87927 100644 --- a/backend/app/routers/projects.py +++ b/backend/app/routers/projects.py @@ -64,6 +64,7 @@ async def get_projects( enable_admin: bool = False, ): + # TODO check if the current user is a member OR creator query = (ProjectDB.creator.email == user_id) projects_and_count = await ProjectDB.find(*query).to_list() From f832ca40561dc34ca8a04f36699ddc6c677b9879 Mon Sep 17 00:00:00 2001 From: toddn Date: Mon, 29 Jul 2024 16:16:05 -0500 Subject: [PATCH 08/28] add a remove project member --- backend/app/routers/projects.py | 278 +++++++++++++++++++++++++++++++- 1 file changed, 277 insertions(+), 1 deletion(-) diff --git a/backend/app/routers/projects.py b/backend/app/routers/projects.py index 2d8c87927..878e882a5 100644 --- a/backend/app/routers/projects.py +++ b/backend/app/routers/projects.py @@ -90,4 +90,280 @@ async def get_project( if project := await ProjectDB.find_one(ProjectDB.id == PydanticObjectId(project_id) ) is not None: return project.dict() - raise HTTPException(status_code=404, detail=f"Project {project_id} not found") \ No newline at end of file + raise HTTPException(status_code=404, detail=f"Project {project_id} not found") + +from datetime import datetime +from typing import Optional + +from app import dependencies +from app.deps.authorization_deps import AuthorizationDB, GroupAuthorization +from app.keycloak_auth import get_current_user, get_user +from app.models.authorization import RoleType +from app.models.datasets import DatasetDB, DatasetOut +from app.models.groups import GroupBase, GroupDB, GroupIn, GroupOut, Member +from app.models.pages import Paged, _construct_page_metadata, _get_page_query +from app.models.users import UserDB, UserOut +from app.routers.authentication import get_admin, get_admin_mode +from app.search.index import index_dataset, index_dataset_files +from beanie import PydanticObjectId +from beanie.operators import Or, Push, RegEx +from bson.objectid import ObjectId +from fastapi import APIRouter, Depends, HTTPException + +router = APIRouter() + + +@router.post("", response_model=GroupOut) +async def save_group( + group_in: GroupIn, + user=Depends(get_current_user), +): + group_db = GroupDB(**group_in.dict(), creator=user.email) + user_member = Member(user=user, editor=True) + if user_member not in group_db.users: + group_db.users.append(user_member) + await group_db.insert() + return group_db.dict() + + +@router.get("", response_model=Paged) +async def get_groups( + user_id=Depends(get_user), + skip: int = 0, + limit: int = 10, + enable_admin: bool = False, + admin_mode: bool = Depends(get_admin_mode), + admin=Depends(get_admin), +): + """Get a list of all Groups in the db the user is a member/owner of. + + Arguments: + skip -- number of initial recoto_list()rds to skip (i.e. for pagination) + limit -- restrict number of records to be returned (i.e. for pagination) + + + """ + criteria_list = [] + if not admin or not admin_mode: + criteria_list.append( + Or( + GroupDB.creator == user_id, + GroupDB.users.user.email == user_id, + ) + ) + + groups_and_count = ( + await GroupDB.find( + *criteria_list, + ) + .aggregate( + [_get_page_query(skip, limit, sort_field="created", ascending=False)], + ) + .to_list() + ) + page_metadata = _construct_page_metadata(groups_and_count, skip, limit) + page = Paged( + metadata=page_metadata, + data=[ + GroupOut(id=item.pop("_id"), **item) for item in groups_and_count[0]["data"] + ], + ) + return page.dict() + + +@router.get("/search/{search_term}", response_model=Paged) +async def search_group( + search_term: str, + user_id=Depends(get_user), + skip: int = 0, + limit: int = 10, + enable_admin: bool = False, + admin_mode: bool = Depends(get_admin_mode), + admin=Depends(get_admin), +): + """Search all groups in the db based on text. + + Arguments: + text -- any text matching name or description + skip -- number of initial records to skip (i.e. for pagination) + limit -- restrict number of records to be returned (i.e. for pagination) + """ + + criteria_list = [ + Or( + RegEx(field=GroupDB.name, pattern=search_term, options="i"), + RegEx(field=GroupDB.description, pattern=search_term, options="i"), + ), + ] + if not admin or not admin_mode: + criteria_list.append( + Or(GroupDB.creator == user_id, GroupDB.users.user.email == user_id) + ) + + # user has to be the creator or member first; then apply search + groups_and_count = ( + await GroupDB.find( + *criteria_list, + ) + .aggregate( + [_get_page_query(skip, limit, sort_field="created", ascending=False)], + ) + .to_list() + ) + page_metadata = _construct_page_metadata(groups_and_count, skip, limit) + page = Paged( + metadata=page_metadata, + data=[ + GroupOut(id=item.pop("_id"), **item) for item in groups_and_count[0]["data"] + ], + ) + return page.dict() + + +@router.get("/{group_id}", response_model=GroupOut) +async def get_group( + group_id: str, + allow: bool = Depends(GroupAuthorization("viewer")), +): + if (group := await GroupDB.get(PydanticObjectId(group_id))) is not None: + return group.dict() + raise HTTPException(status_code=404, detail=f"Group {group_id} not found") + + +@router.put("/{group_id}", response_model=GroupOut) +async def edit_group( + group_id: str, + group_info: GroupBase, + user_id=Depends(get_user), + allow: bool = Depends(GroupAuthorization("editor")), +): + if (group := await GroupDB.get(PydanticObjectId(group_id))) is not None: + group_dict = dict(group_info) if group_info is not None else {} + + if len(group_dict["name"]) == 0 or len(group_dict["users"]) == 0: + raise HTTPException( + status_code=400, + detail="Group name can't be null or user list can't be empty", + ) + return + + user = await UserDB.find_one(UserDB.email == user_id) + group_dict["creator"] = user.dict() + group_dict["modified"] = datetime.utcnow() + groups_users = group_dict["users"] + original_users = group.users + + # remove users that are no longer in this group + for original_user in original_users: + if original_user not in groups_users: + # remove them from auth + async for auth in AuthorizationDB.find( + {"group_ids": ObjectId(group_id)} + ): + auth.user_ids.remove(original_user.user.email) + await auth.replace() + # Update group itself + group.users.remove(original_user) + await group.replace() + # add new users to the group + for i in range(0, len(groups_users)): + user = groups_users[i] + if user in group.users: + for original_user in group.users: + if original_user.user.email == user.user.email: + original_editor = original_user.editor + new_editor = user.editor + # replace the user if editor has changed + if not new_editor == original_editor: + group.users.remove(original_user) + group.users.append(user) + await group.replace() + else: + # if user is not in the group add user + group.users.append(user) + await group.replace() + # Add user to all affected Authorization entries + await AuthorizationDB.find( + AuthorizationDB.group_ids == ObjectId(group_id), + ).update( + Push({AuthorizationDB.user_ids: user.email}), + ) + try: + group.name = group_dict["name"] + await group.replace() + if "description" in group_dict: + group.description = group_dict["description"] + await group.replace() + except Exception as e: + raise HTTPException(status_code=500, detail=e.args[0]) + return group.dict() + raise HTTPException(status_code=404, detail=f"Group {group_id} not found") + + +@router.delete("/{group_id}", response_model=GroupOut) +async def delete_group( + group_id: str, + allow: bool = Depends(GroupAuthorization("owner")), +): + if (group := await GroupDB.get(PydanticObjectId(group_id))) is not None: + await group.delete() + return group.dict() # TODO: Do we need to return what we just deleted? + else: + raise HTTPException(status_code=404, detail=f"Dataset {group_id} not found") + + +@router.post("/{project_id}/add/{username}", response_model=ProjectOut) +async def add_member( + project_id: str, + username: str, + role: Optional[str] = None, + es=Depends(dependencies.get_elasticsearchclient), + allow: bool = Depends(GroupAuthorization("editor")), +): + """Add a new user to a group.""" + if (user := await UserDB.find_one(UserDB.email == username)) is not None: + new_member = Member(user=UserOut(**user.dict())) + if (project := await ProjectDB.get(PydanticObjectId(project_id))) is not None: + found_already = False + for u in project.users: + if u.user.email == username: + found_already = True + break + if not found_already: + # If user is already in the group, skip directly to returning the group + # else add role and attach this member + + if role is not None and role == RoleType.EDITOR: + new_member.editor = True + else: + new_member.editor = False + project.users.append(new_member) + await project.replace() + return project.dict() + raise HTTPException(status_code=404, detail=f"Group {project_id} not found") + raise HTTPException(status_code=404, detail=f"User {username} not found") + +@router.post("/{project_id}/remove/{username}", response_model=ProjectOut) +async def remove_member( + project_id: str, + username: str, + es=Depends(dependencies.get_elasticsearchclient), + allow: bool = Depends(GroupAuthorization("editor")), +): + """Remove a user from a group.""" + + if (project := await ProjectDB.get(PydanticObjectId(project_id))) is not None: + # Is the user actually in the group already? + found_user = None + for u in project.users: + if u.user.email == username: + found_user = u + if not found_user: + # TODO: User wasn't in group, should this throw an error instead? Either way, the user is removed... + return project + # Update group itself + project.users.remove(found_user) + await project.replace() + return project.dict() + raise HTTPException(status_code=404, detail=f"Group {project_id} not found") + From 2c9a0505692f6a26e64d4bcdd8b7e1f4aae7b132 Mon Sep 17 00:00:00 2001 From: toddn Date: Mon, 29 Jul 2024 16:20:26 -0500 Subject: [PATCH 09/28] routes for deleting and adding, removing members --- backend/app/routers/projects.py | 235 ++------------------------------ 1 file changed, 9 insertions(+), 226 deletions(-) diff --git a/backend/app/routers/projects.py b/backend/app/routers/projects.py index 878e882a5..20dfa6749 100644 --- a/backend/app/routers/projects.py +++ b/backend/app/routers/projects.py @@ -12,12 +12,13 @@ from app import dependencies from app.config import settings - +from app.models.users import UserDB, UserOut, UserIn from app.models.project import ( ProjectBase, ProjectDB, ProjectIn, ProjectOut, + Member, ) from app.keycloak_auth import get_current_user, get_token, get_user from beanie import PydanticObjectId @@ -92,224 +93,15 @@ async def get_project( return project.dict() raise HTTPException(status_code=404, detail=f"Project {project_id} not found") -from datetime import datetime -from typing import Optional - -from app import dependencies -from app.deps.authorization_deps import AuthorizationDB, GroupAuthorization -from app.keycloak_auth import get_current_user, get_user -from app.models.authorization import RoleType -from app.models.datasets import DatasetDB, DatasetOut -from app.models.groups import GroupBase, GroupDB, GroupIn, GroupOut, Member -from app.models.pages import Paged, _construct_page_metadata, _get_page_query -from app.models.users import UserDB, UserOut -from app.routers.authentication import get_admin, get_admin_mode -from app.search.index import index_dataset, index_dataset_files -from beanie import PydanticObjectId -from beanie.operators import Or, Push, RegEx -from bson.objectid import ObjectId -from fastapi import APIRouter, Depends, HTTPException - -router = APIRouter() - - -@router.post("", response_model=GroupOut) -async def save_group( - group_in: GroupIn, - user=Depends(get_current_user), -): - group_db = GroupDB(**group_in.dict(), creator=user.email) - user_member = Member(user=user, editor=True) - if user_member not in group_db.users: - group_db.users.append(user_member) - await group_db.insert() - return group_db.dict() - - -@router.get("", response_model=Paged) -async def get_groups( - user_id=Depends(get_user), - skip: int = 0, - limit: int = 10, - enable_admin: bool = False, - admin_mode: bool = Depends(get_admin_mode), - admin=Depends(get_admin), -): - """Get a list of all Groups in the db the user is a member/owner of. - - Arguments: - skip -- number of initial recoto_list()rds to skip (i.e. for pagination) - limit -- restrict number of records to be returned (i.e. for pagination) - - - """ - criteria_list = [] - if not admin or not admin_mode: - criteria_list.append( - Or( - GroupDB.creator == user_id, - GroupDB.users.user.email == user_id, - ) - ) - - groups_and_count = ( - await GroupDB.find( - *criteria_list, - ) - .aggregate( - [_get_page_query(skip, limit, sort_field="created", ascending=False)], - ) - .to_list() - ) - page_metadata = _construct_page_metadata(groups_and_count, skip, limit) - page = Paged( - metadata=page_metadata, - data=[ - GroupOut(id=item.pop("_id"), **item) for item in groups_and_count[0]["data"] - ], - ) - return page.dict() - - -@router.get("/search/{search_term}", response_model=Paged) -async def search_group( - search_term: str, - user_id=Depends(get_user), - skip: int = 0, - limit: int = 10, - enable_admin: bool = False, - admin_mode: bool = Depends(get_admin_mode), - admin=Depends(get_admin), -): - """Search all groups in the db based on text. - - Arguments: - text -- any text matching name or description - skip -- number of initial records to skip (i.e. for pagination) - limit -- restrict number of records to be returned (i.e. for pagination) - """ - - criteria_list = [ - Or( - RegEx(field=GroupDB.name, pattern=search_term, options="i"), - RegEx(field=GroupDB.description, pattern=search_term, options="i"), - ), - ] - if not admin or not admin_mode: - criteria_list.append( - Or(GroupDB.creator == user_id, GroupDB.users.user.email == user_id) - ) - - # user has to be the creator or member first; then apply search - groups_and_count = ( - await GroupDB.find( - *criteria_list, - ) - .aggregate( - [_get_page_query(skip, limit, sort_field="created", ascending=False)], - ) - .to_list() - ) - page_metadata = _construct_page_metadata(groups_and_count, skip, limit) - page = Paged( - metadata=page_metadata, - data=[ - GroupOut(id=item.pop("_id"), **item) for item in groups_and_count[0]["data"] - ], - ) - return page.dict() - - -@router.get("/{group_id}", response_model=GroupOut) -async def get_group( - group_id: str, - allow: bool = Depends(GroupAuthorization("viewer")), -): - if (group := await GroupDB.get(PydanticObjectId(group_id))) is not None: - return group.dict() - raise HTTPException(status_code=404, detail=f"Group {group_id} not found") - - -@router.put("/{group_id}", response_model=GroupOut) -async def edit_group( - group_id: str, - group_info: GroupBase, - user_id=Depends(get_user), - allow: bool = Depends(GroupAuthorization("editor")), -): - if (group := await GroupDB.get(PydanticObjectId(group_id))) is not None: - group_dict = dict(group_info) if group_info is not None else {} - - if len(group_dict["name"]) == 0 or len(group_dict["users"]) == 0: - raise HTTPException( - status_code=400, - detail="Group name can't be null or user list can't be empty", - ) - return - - user = await UserDB.find_one(UserDB.email == user_id) - group_dict["creator"] = user.dict() - group_dict["modified"] = datetime.utcnow() - groups_users = group_dict["users"] - original_users = group.users - - # remove users that are no longer in this group - for original_user in original_users: - if original_user not in groups_users: - # remove them from auth - async for auth in AuthorizationDB.find( - {"group_ids": ObjectId(group_id)} - ): - auth.user_ids.remove(original_user.user.email) - await auth.replace() - # Update group itself - group.users.remove(original_user) - await group.replace() - # add new users to the group - for i in range(0, len(groups_users)): - user = groups_users[i] - if user in group.users: - for original_user in group.users: - if original_user.user.email == user.user.email: - original_editor = original_user.editor - new_editor = user.editor - # replace the user if editor has changed - if not new_editor == original_editor: - group.users.remove(original_user) - group.users.append(user) - await group.replace() - else: - # if user is not in the group add user - group.users.append(user) - await group.replace() - # Add user to all affected Authorization entries - await AuthorizationDB.find( - AuthorizationDB.group_ids == ObjectId(group_id), - ).update( - Push({AuthorizationDB.user_ids: user.email}), - ) - try: - group.name = group_dict["name"] - await group.replace() - if "description" in group_dict: - group.description = group_dict["description"] - await group.replace() - except Exception as e: - raise HTTPException(status_code=500, detail=e.args[0]) - return group.dict() - raise HTTPException(status_code=404, detail=f"Group {group_id} not found") - - -@router.delete("/{group_id}", response_model=GroupOut) -async def delete_group( - group_id: str, - allow: bool = Depends(GroupAuthorization("owner")), +@router.delete("/{project_id}", response_model=ProjectOut) +async def delete_project( + project_id: str, ): - if (group := await GroupDB.get(PydanticObjectId(group_id))) is not None: - await group.delete() - return group.dict() # TODO: Do we need to return what we just deleted? + if (project := await ProjectDB.get(PydanticObjectId(project_id))) is not None: + await project.delete() + return project.dict() # TODO: Do we need to return what we just deleted? else: - raise HTTPException(status_code=404, detail=f"Dataset {group_id} not found") + raise HTTPException(status_code=404, detail=f"Project {project_id} not found") @router.post("/{project_id}/add/{username}", response_model=ProjectOut) @@ -317,8 +109,6 @@ async def add_member( project_id: str, username: str, role: Optional[str] = None, - es=Depends(dependencies.get_elasticsearchclient), - allow: bool = Depends(GroupAuthorization("editor")), ): """Add a new user to a group.""" if (user := await UserDB.find_one(UserDB.email == username)) is not None: @@ -332,11 +122,6 @@ async def add_member( if not found_already: # If user is already in the group, skip directly to returning the group # else add role and attach this member - - if role is not None and role == RoleType.EDITOR: - new_member.editor = True - else: - new_member.editor = False project.users.append(new_member) await project.replace() return project.dict() @@ -347,8 +132,6 @@ async def add_member( async def remove_member( project_id: str, username: str, - es=Depends(dependencies.get_elasticsearchclient), - allow: bool = Depends(GroupAuthorization("editor")), ): """Remove a user from a group.""" From 662d9b5fb420aa306011aaed26527123f2b078d9 Mon Sep 17 00:00:00 2001 From: toddn Date: Mon, 29 Jul 2024 16:20:59 -0500 Subject: [PATCH 10/28] "project" not "group" --- backend/app/routers/projects.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/app/routers/projects.py b/backend/app/routers/projects.py index 20dfa6749..d13336a5b 100644 --- a/backend/app/routers/projects.py +++ b/backend/app/routers/projects.py @@ -125,7 +125,7 @@ async def add_member( project.users.append(new_member) await project.replace() return project.dict() - raise HTTPException(status_code=404, detail=f"Group {project_id} not found") + raise HTTPException(status_code=404, detail=f"Project {project_id} not found") raise HTTPException(status_code=404, detail=f"User {username} not found") @router.post("/{project_id}/remove/{username}", response_model=ProjectOut) @@ -148,5 +148,5 @@ async def remove_member( project.users.remove(found_user) await project.replace() return project.dict() - raise HTTPException(status_code=404, detail=f"Group {project_id} not found") + raise HTTPException(status_code=404, detail=f"Project {project_id} not found") From f116dc40e664b493f27d64ef9cf8ae70cbb1ed9d Mon Sep 17 00:00:00 2001 From: toddn Date: Tue, 30 Jul 2024 09:11:41 -0500 Subject: [PATCH 11/28] methods for adding and removing datasets from projects folders and files will be the same --- backend/app/routers/projects.py | 36 +++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/backend/app/routers/projects.py b/backend/app/routers/projects.py index d13336a5b..ac4894ee8 100644 --- a/backend/app/routers/projects.py +++ b/backend/app/routers/projects.py @@ -13,6 +13,7 @@ from app import dependencies from app.config import settings from app.models.users import UserDB, UserOut, UserIn +from app.models.datasets import DatasetDB from app.models.project import ( ProjectBase, ProjectDB, @@ -56,6 +57,41 @@ async def save_project( # TODO Add new entry to elasticsearch return project.dict() +@router.post("/{project_id}/add_dataset/{dataset_id}", response_model=ProjectOut) +async def add_dataset( + project_id: str, + dataset_id: str, +): + if project := await ProjectDB.find_one(ProjectDB.id == PydanticObjectId(project_id) + ) is not None: + if dataset := await DatasetDB.find_one(DatasetDB.id == PydanticObjectId(dataset_id) + ) is not None: + project.dataset_ids.append(dataset_id) + await project.replace() + return project.dict() + raise HTTPException(status_code=404, detail=f"Dataset {dataset_id} not found") + raise HTTPException(status_code=404, detail=f"Project {project_id} not found") + +@router.post("/{project_id}/remove_dataset/{dataset_id}", response_model=ProjectOut) +async def add_dataset( + project_id: str, + dataset_id: str, +): + if project := await ProjectDB.find_one(ProjectDB.id == PydanticObjectId(project_id) + ) is not None: + if dataset := await DatasetDB.find_one(DatasetDB.id == PydanticObjectId(dataset_id) + ) is not None: + if dataset_id in project.dataset_ids: + project.dataset_ids.remove(dataset_id) + await project.replace() + return project.dict() + else: + return project.dict() + raise HTTPException(status_code=404, detail=f"Dataset {dataset_id} not found") + raise HTTPException(status_code=404, detail=f"Project {project_id} not found") + + + @router.get("", response_model=Paged) async def get_projects( user_id=Depends(get_user), From 3c371a130fadac71f89b1ae10d48df17d807c27a Mon Sep 17 00:00:00 2001 From: toddn Date: Tue, 30 Jul 2024 09:14:51 -0500 Subject: [PATCH 12/28] remove license --- backend/app/routers/projects.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/backend/app/routers/projects.py b/backend/app/routers/projects.py index ac4894ee8..1dd5fab60 100644 --- a/backend/app/routers/projects.py +++ b/backend/app/routers/projects.py @@ -43,7 +43,6 @@ @router.post("", response_model=ProjectOut) async def save_project( project_in: ProjectIn, - license_id: str, user=Depends(get_current_user), es: Elasticsearch = Depends(dependencies.get_elasticsearchclient), ): @@ -73,7 +72,7 @@ async def add_dataset( raise HTTPException(status_code=404, detail=f"Project {project_id} not found") @router.post("/{project_id}/remove_dataset/{dataset_id}", response_model=ProjectOut) -async def add_dataset( +async def remove_dataset( project_id: str, dataset_id: str, ): From ceb9b1a7e7ae2404956f6b4e0ed887d4385bb0ff Mon Sep 17 00:00:00 2001 From: toddn Date: Tue, 30 Jul 2024 09:22:02 -0500 Subject: [PATCH 13/28] adding and removing folders and files --- backend/app/routers/projects.py | 68 +++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/backend/app/routers/projects.py b/backend/app/routers/projects.py index 1dd5fab60..037c86682 100644 --- a/backend/app/routers/projects.py +++ b/backend/app/routers/projects.py @@ -14,6 +14,8 @@ from app.config import settings from app.models.users import UserDB, UserOut, UserIn from app.models.datasets import DatasetDB +from app.models.files import FileDB +from app.models.folders import FolderDB from app.models.project import ( ProjectBase, ProjectDB, @@ -90,6 +92,72 @@ async def remove_dataset( raise HTTPException(status_code=404, detail=f"Project {project_id} not found") +@router.post("/{project_id}/add_folder/{folder_id}", response_model=ProjectOut) +async def add_folder( + project_id: str, + folder_id: str, +): + if project := await ProjectDB.find_one(ProjectDB.id == PydanticObjectId(project_id) + ) is not None: + if folder := await FolderDB.find_one(FolderDB.id == PydanticObjectId(dataset_id) + ) is not None: + project.folder_ids.append(folder_id) + await project.replace() + return project.dict() + raise HTTPException(status_code=404, detail=f"Folder {folder_id} not found") + raise HTTPException(status_code=404, detail=f"Project {project_id} not found") + +@router.post("/{project_id}/remove_folder/{folder_id}", response_model=ProjectOut) +async def remove_folder( + project_id: str, + folder_id: str, +): + if project := await ProjectDB.find_one(ProjectDB.id == PydanticObjectId(project_id) + ) is not None: + if folder := await FolderDB.find_one(FolderDB.id == PydanticObjectId(folder_id) + ) is not None: + if folder_id in project.folder_ids: + project.folder_ids.remove(folder_id) + await project.replace() + return project.dict() + else: + return project.dict() + raise HTTPException(status_code=404, detail=f"Folder {folder_id} not found") + raise HTTPException(status_code=404, detail=f"Project {project_id} not found") + +@router.post("/{project_id}/add_file/{file_id}", response_model=ProjectOut) +async def add_file( + project_id: str, + file_id: str, +): + if project := await ProjectDB.find_one(ProjectDB.id == PydanticObjectId(project_id) + ) is not None: + if file := await FileDB.find_one(FileDB.id == PydanticObjectId(file_id) + ) is not None: + project.file_ids.append(file_id) + await project.replace() + return project.dict() + raise HTTPException(status_code=404, detail=f"File {file_id} not found") + raise HTTPException(status_code=404, detail=f"Project {project_id} not found") + +@router.post("/{project_id}/remove_file/{file_id}", response_model=ProjectOut) +async def remove_file( + project_id: str, + file_id: str, +): + if project := await ProjectDB.find_one(ProjectDB.id == PydanticObjectId(project_id) + ) is not None: + if file := await FileDB.find_one(FileDB.id == PydanticObjectId(file_id) + ) is not None: + if file_id in project.file_ids: + project.file_ids.remove(file_id) + await project.replace() + return project.dict() + else: + return project.dict() + raise HTTPException(status_code=404, detail=f"File {file_id} not found") + raise HTTPException(status_code=404, detail=f"Project {project_id} not found") + @router.get("", response_model=Paged) async def get_projects( From d6f68dd7db3c64ba27dc52c81443a16a8d78ba02 Mon Sep 17 00:00:00 2001 From: toddn Date: Tue, 30 Jul 2024 09:26:55 -0500 Subject: [PATCH 14/28] only add if a file, folder, dataset id not in the project already --- backend/app/routers/projects.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/backend/app/routers/projects.py b/backend/app/routers/projects.py index 037c86682..d37982279 100644 --- a/backend/app/routers/projects.py +++ b/backend/app/routers/projects.py @@ -67,8 +67,9 @@ async def add_dataset( ) is not None: if dataset := await DatasetDB.find_one(DatasetDB.id == PydanticObjectId(dataset_id) ) is not None: - project.dataset_ids.append(dataset_id) - await project.replace() + if dataset_id not in project.dataset_ids: + project.dataset_ids.append(dataset_id) + await project.replace() return project.dict() raise HTTPException(status_code=404, detail=f"Dataset {dataset_id} not found") raise HTTPException(status_code=404, detail=f"Project {project_id} not found") @@ -99,10 +100,11 @@ async def add_folder( ): if project := await ProjectDB.find_one(ProjectDB.id == PydanticObjectId(project_id) ) is not None: - if folder := await FolderDB.find_one(FolderDB.id == PydanticObjectId(dataset_id) + if folder := await FolderDB.find_one(FolderDB.id == PydanticObjectId(folder_id) ) is not None: - project.folder_ids.append(folder_id) - await project.replace() + if folder_id not in project.folder_ids: + project.folder_ids.append(folder_id) + await project.replace() return project.dict() raise HTTPException(status_code=404, detail=f"Folder {folder_id} not found") raise HTTPException(status_code=404, detail=f"Project {project_id} not found") @@ -134,8 +136,9 @@ async def add_file( ) is not None: if file := await FileDB.find_one(FileDB.id == PydanticObjectId(file_id) ) is not None: - project.file_ids.append(file_id) - await project.replace() + if file_id not in project.file_ids: + project.file_ids.append(file_id) + await project.replace() return project.dict() raise HTTPException(status_code=404, detail=f"File {file_id} not found") raise HTTPException(status_code=404, detail=f"Project {project_id} not found") From 715ba8341df97b83a24f879c38e630841d972c5c Mon Sep 17 00:00:00 2001 From: toddn Date: Tue, 30 Jul 2024 09:44:44 -0500 Subject: [PATCH 15/28] beginning project tests formatting --- backend/app/models/project.py | 6 +- backend/app/routers/projects.py | 142 ++++++++++++++++++++--------- backend/app/tests/test_projects.py | 50 ++++++++++ backend/app/tests/utils.py | 16 ++++ 4 files changed, 169 insertions(+), 45 deletions(-) create mode 100644 backend/app/tests/test_projects.py diff --git a/backend/app/models/project.py b/backend/app/models/project.py index adaa9a7ff..52b964696 100644 --- a/backend/app/models/project.py +++ b/backend/app/models/project.py @@ -14,6 +14,7 @@ class Member(BaseModel): user: UserOut editor: bool = False + class ProjectBase(BaseModel): id: PydanticObjectId = Field(default_factory=PydanticObjectId, alias="_id") name: str @@ -26,13 +27,16 @@ class ProjectBase(BaseModel): creator: UserOut users: List[Member] = [] + class ProjectDB(Document, ProjectBase): class Settings: name = "projects" + class ProjectIn(ProjectBase): pass + class ProjectOut(ProjectDB): class Config: - fields = {"id": "id"} \ No newline at end of file + fields = {"id": "id"} diff --git a/backend/app/routers/projects.py b/backend/app/routers/projects.py index d37982279..2c0eeb84c 100644 --- a/backend/app/routers/projects.py +++ b/backend/app/routers/projects.py @@ -42,13 +42,13 @@ clowder_bucket = os.getenv("MINIO_BUCKET_NAME", "clowder") + @router.post("", response_model=ProjectOut) async def save_project( project_in: ProjectIn, user=Depends(get_current_user), es: Elasticsearch = Depends(dependencies.get_elasticsearchclient), ): - project = ProjectDB( **project_in.dict(), creator=user, @@ -58,15 +58,24 @@ async def save_project( # TODO Add new entry to elasticsearch return project.dict() + @router.post("/{project_id}/add_dataset/{dataset_id}", response_model=ProjectOut) async def add_dataset( - project_id: str, - dataset_id: str, + project_id: str, + dataset_id: str, ): - if project := await ProjectDB.find_one(ProjectDB.id == PydanticObjectId(project_id) - ) is not None: - if dataset := await DatasetDB.find_one(DatasetDB.id == PydanticObjectId(dataset_id) - ) is not None: + if ( + project := await ProjectDB.find_one( + ProjectDB.id == PydanticObjectId(project_id) + ) + is not None + ): + if ( + dataset := await DatasetDB.find_one( + DatasetDB.id == PydanticObjectId(dataset_id) + ) + is not None + ): if dataset_id not in project.dataset_ids: project.dataset_ids.append(dataset_id) await project.replace() @@ -74,15 +83,24 @@ async def add_dataset( raise HTTPException(status_code=404, detail=f"Dataset {dataset_id} not found") raise HTTPException(status_code=404, detail=f"Project {project_id} not found") + @router.post("/{project_id}/remove_dataset/{dataset_id}", response_model=ProjectOut) async def remove_dataset( - project_id: str, - dataset_id: str, + project_id: str, + dataset_id: str, ): - if project := await ProjectDB.find_one(ProjectDB.id == PydanticObjectId(project_id) - ) is not None: - if dataset := await DatasetDB.find_one(DatasetDB.id == PydanticObjectId(dataset_id) - ) is not None: + if ( + project := await ProjectDB.find_one( + ProjectDB.id == PydanticObjectId(project_id) + ) + is not None + ): + if ( + dataset := await DatasetDB.find_one( + DatasetDB.id == PydanticObjectId(dataset_id) + ) + is not None + ): if dataset_id in project.dataset_ids: project.dataset_ids.remove(dataset_id) await project.replace() @@ -95,13 +113,21 @@ async def remove_dataset( @router.post("/{project_id}/add_folder/{folder_id}", response_model=ProjectOut) async def add_folder( - project_id: str, - folder_id: str, + project_id: str, + folder_id: str, ): - if project := await ProjectDB.find_one(ProjectDB.id == PydanticObjectId(project_id) - ) is not None: - if folder := await FolderDB.find_one(FolderDB.id == PydanticObjectId(folder_id) - ) is not None: + if ( + project := await ProjectDB.find_one( + ProjectDB.id == PydanticObjectId(project_id) + ) + is not None + ): + if ( + folder := await FolderDB.find_one( + FolderDB.id == PydanticObjectId(folder_id) + ) + is not None + ): if folder_id not in project.folder_ids: project.folder_ids.append(folder_id) await project.replace() @@ -109,15 +135,24 @@ async def add_folder( raise HTTPException(status_code=404, detail=f"Folder {folder_id} not found") raise HTTPException(status_code=404, detail=f"Project {project_id} not found") + @router.post("/{project_id}/remove_folder/{folder_id}", response_model=ProjectOut) async def remove_folder( - project_id: str, - folder_id: str, + project_id: str, + folder_id: str, ): - if project := await ProjectDB.find_one(ProjectDB.id == PydanticObjectId(project_id) - ) is not None: - if folder := await FolderDB.find_one(FolderDB.id == PydanticObjectId(folder_id) - ) is not None: + if ( + project := await ProjectDB.find_one( + ProjectDB.id == PydanticObjectId(project_id) + ) + is not None + ): + if ( + folder := await FolderDB.find_one( + FolderDB.id == PydanticObjectId(folder_id) + ) + is not None + ): if folder_id in project.folder_ids: project.folder_ids.remove(folder_id) await project.replace() @@ -127,15 +162,22 @@ async def remove_folder( raise HTTPException(status_code=404, detail=f"Folder {folder_id} not found") raise HTTPException(status_code=404, detail=f"Project {project_id} not found") + @router.post("/{project_id}/add_file/{file_id}", response_model=ProjectOut) async def add_file( - project_id: str, - file_id: str, + project_id: str, + file_id: str, ): - if project := await ProjectDB.find_one(ProjectDB.id == PydanticObjectId(project_id) - ) is not None: - if file := await FileDB.find_one(FileDB.id == PydanticObjectId(file_id) - ) is not None: + if ( + project := await ProjectDB.find_one( + ProjectDB.id == PydanticObjectId(project_id) + ) + is not None + ): + if ( + file := await FileDB.find_one(FileDB.id == PydanticObjectId(file_id)) + is not None + ): if file_id not in project.file_ids: project.file_ids.append(file_id) await project.replace() @@ -143,15 +185,22 @@ async def add_file( raise HTTPException(status_code=404, detail=f"File {file_id} not found") raise HTTPException(status_code=404, detail=f"Project {project_id} not found") + @router.post("/{project_id}/remove_file/{file_id}", response_model=ProjectOut) async def remove_file( - project_id: str, - file_id: str, + project_id: str, + file_id: str, ): - if project := await ProjectDB.find_one(ProjectDB.id == PydanticObjectId(project_id) - ) is not None: - if file := await FileDB.find_one(FileDB.id == PydanticObjectId(file_id) - ) is not None: + if ( + project := await ProjectDB.find_one( + ProjectDB.id == PydanticObjectId(project_id) + ) + is not None + ): + if ( + file := await FileDB.find_one(FileDB.id == PydanticObjectId(file_id)) + is not None + ): if file_id in project.file_ids: project.file_ids.remove(file_id) await project.replace() @@ -170,9 +219,8 @@ async def get_projects( mine: bool = False, enable_admin: bool = False, ): - # TODO check if the current user is a member OR creator - query = (ProjectDB.creator.email == user_id) + query = ProjectDB.creator.email == user_id projects_and_count = await ProjectDB.find(*query).to_list() @@ -190,15 +238,21 @@ async def get_projects( return page.dict() + @router.get("/{project_id}", response_model=ProjectOut) async def get_project( project_id: str, ): - if project := await ProjectDB.find_one(ProjectDB.id == PydanticObjectId(project_id) - ) is not None: + if ( + project := await ProjectDB.find_one( + ProjectDB.id == PydanticObjectId(project_id) + ) + is not None + ): return project.dict() raise HTTPException(status_code=404, detail=f"Project {project_id} not found") + @router.delete("/{project_id}", response_model=ProjectOut) async def delete_project( project_id: str, @@ -210,7 +264,7 @@ async def delete_project( raise HTTPException(status_code=404, detail=f"Project {project_id} not found") -@router.post("/{project_id}/add/{username}", response_model=ProjectOut) +@router.post("/{project_id}/add_member/{username}", response_model=ProjectOut) async def add_member( project_id: str, username: str, @@ -234,7 +288,8 @@ async def add_member( raise HTTPException(status_code=404, detail=f"Project {project_id} not found") raise HTTPException(status_code=404, detail=f"User {username} not found") -@router.post("/{project_id}/remove/{username}", response_model=ProjectOut) + +@router.post("/{project_id}/remove_member/{username}", response_model=ProjectOut) async def remove_member( project_id: str, username: str, @@ -255,4 +310,3 @@ async def remove_member( await project.replace() return project.dict() raise HTTPException(status_code=404, detail=f"Project {project_id} not found") - diff --git a/backend/app/tests/test_projects.py b/backend/app/tests/test_projects.py new file mode 100644 index 000000000..0196e0dec --- /dev/null +++ b/backend/app/tests/test_projects.py @@ -0,0 +1,50 @@ +from app.config import settings +from app.tests.utils import ( + create_dataset, + create_group, + create_user, + get_user_token, + user_alt, + create_project, +) +from fastapi.testclient import TestClient + +member_alt = {"user": user_alt, "editor": False} + + +def test_create_project(client: TestClient, headers: dict): + create_project(client, headers) + + +def test_get_project(client: TestClient, headers: dict): + project_id = create_project(client, headers).get("id") + response = client.get(f"{settings.API_V2_STR}/projects/{project_id}", headers=headers) + assert response.status_code == 200 + assert response.json().get("id") is not None + + +def test_delete_project(client: TestClient, headers: dict): + project_id = create_project(client, headers).get("id") + response = client.delete( + f"{settings.API_V2_STR}/projects/{project_id}", headers=headers + ) + assert response.status_code == 200 + + + +def test_add_member(client: TestClient, headers: dict): + new_project = create_project(client, headers) + project_id = new_project.get("id") + + create_user(client, headers) + new_project["users"].append(member_alt) + + response = client.post( + f"{settings.API_V2_STR}/projects/{project_id}/add_member/{member_alt['user']['email']}", + headers=headers, + ) + + assert response.status_code == 200 + assert response.json().get("id") is not None + for user in response.json().get("users"): + assert user.get("user").get("email") == member_alt["user"]["email"] diff --git a/backend/app/tests/utils.py b/backend/app/tests/utils.py index 395d2a676..341ffd516 100644 --- a/backend/app/tests/utils.py +++ b/backend/app/tests/utils.py @@ -61,6 +61,11 @@ }, } +project_example = { + "name": "test_project", + "description": "This project is a test" +} + extractor_info_v1_example = { "@context": "http://clowder.ncsa.illinois.edu/contexts/extractors.jsonld", "name": "test.extractor_info_v1_example", @@ -175,6 +180,17 @@ def create_dataset_with_custom_license(client: TestClient, headers: dict): assert response.json().get("id") is not None return response.json() +def create_project(client: TestClient, headers: dict): + """Creates a test dataset and returns the JSON.""" + response = client.post( + f"{settings.API_V2_STR}/projects", + headers=headers, + json=project_example, + ) + assert response.status_code == 200 + assert response.json().get("id") is not None + return response.json() + def upload_file( client: TestClient, From 01e16e983c9c18f21d8ad6c7183a6c2143c35d91 Mon Sep 17 00:00:00 2001 From: toddn Date: Tue, 30 Jul 2024 09:46:38 -0500 Subject: [PATCH 16/28] add test add dataset to project --- backend/app/tests/test_projects.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/backend/app/tests/test_projects.py b/backend/app/tests/test_projects.py index 0196e0dec..f734c4352 100644 --- a/backend/app/tests/test_projects.py +++ b/backend/app/tests/test_projects.py @@ -48,3 +48,19 @@ def test_add_member(client: TestClient, headers: dict): assert response.json().get("id") is not None for user in response.json().get("users"): assert user.get("user").get("email") == member_alt["user"]["email"] + +def test_add_dataset(client: TestClient, headers: dict): + new_project = create_project(client, headers) + project_id = new_project.get("id") + + dataset_id = create_dataset(client, headers).get("id") + + response = client.post( + f"{settings.API_V2_STR}/projects/{project_id}/add_dataset/{dataset_id}", + headers=headers, + ) + + assert response.status_code == 200 + assert response.json().get("id") is not None + assert dataset_id in response.json().get("dataset_ids") + From c2024c261b1e12542de85dc4af220c65e24c5825 Mon Sep 17 00:00:00 2001 From: toddn Date: Tue, 30 Jul 2024 09:47:13 -0500 Subject: [PATCH 17/28] formatting --- backend/app/tests/test_projects.py | 7 ++++--- backend/app/tests/utils.py | 6 ++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/backend/app/tests/test_projects.py b/backend/app/tests/test_projects.py index f734c4352..e96420b26 100644 --- a/backend/app/tests/test_projects.py +++ b/backend/app/tests/test_projects.py @@ -18,7 +18,9 @@ def test_create_project(client: TestClient, headers: dict): def test_get_project(client: TestClient, headers: dict): project_id = create_project(client, headers).get("id") - response = client.get(f"{settings.API_V2_STR}/projects/{project_id}", headers=headers) + response = client.get( + f"{settings.API_V2_STR}/projects/{project_id}", headers=headers + ) assert response.status_code == 200 assert response.json().get("id") is not None @@ -31,7 +33,6 @@ def test_delete_project(client: TestClient, headers: dict): assert response.status_code == 200 - def test_add_member(client: TestClient, headers: dict): new_project = create_project(client, headers) project_id = new_project.get("id") @@ -49,6 +50,7 @@ def test_add_member(client: TestClient, headers: dict): for user in response.json().get("users"): assert user.get("user").get("email") == member_alt["user"]["email"] + def test_add_dataset(client: TestClient, headers: dict): new_project = create_project(client, headers) project_id = new_project.get("id") @@ -63,4 +65,3 @@ def test_add_dataset(client: TestClient, headers: dict): assert response.status_code == 200 assert response.json().get("id") is not None assert dataset_id in response.json().get("dataset_ids") - diff --git a/backend/app/tests/utils.py b/backend/app/tests/utils.py index 341ffd516..e6fedec52 100644 --- a/backend/app/tests/utils.py +++ b/backend/app/tests/utils.py @@ -61,10 +61,7 @@ }, } -project_example = { - "name": "test_project", - "description": "This project is a test" -} +project_example = {"name": "test_project", "description": "This project is a test"} extractor_info_v1_example = { "@context": "http://clowder.ncsa.illinois.edu/contexts/extractors.jsonld", @@ -180,6 +177,7 @@ def create_dataset_with_custom_license(client: TestClient, headers: dict): assert response.json().get("id") is not None return response.json() + def create_project(client: TestClient, headers: dict): """Creates a test dataset and returns the JSON.""" response = client.post( From c6073f6b6ed9d780c56297f00062db2cdddd300a Mon Sep 17 00:00:00 2001 From: toddn Date: Tue, 30 Jul 2024 10:15:44 -0500 Subject: [PATCH 18/28] adding routes to main fixing create and get routes --- backend/app/main.py | 8 ++++++++ backend/app/routers/projects.py | 9 ++++----- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/backend/app/main.py b/backend/app/main.py index 84f54d38b..f40c4172a 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -39,6 +39,7 @@ VisualizationDataDBViewList, VisualizationDataFreezeDB, ) +from app.models.project import ProjectDB from app.routers import ( authentication, authorization, @@ -66,6 +67,7 @@ thumbnails, users, visualization, + projects, ) # setup loggers @@ -250,6 +252,11 @@ prefix="/public_thumbnails", tags=["public_thumbnails"], ) +api_router.include_router( + projects.router, + prefix="/projects", + tags=["projects"], +) api_router.include_router( licenses.router, prefix="/licenses", @@ -315,6 +322,7 @@ async def startup_beanie(): ThumbnailFreezeDB, ThumbnailDBViewList, LicenseDB, + ProjectDB, ], recreate_views=True, ) diff --git a/backend/app/routers/projects.py b/backend/app/routers/projects.py index 2c0eeb84c..3d57a9aab 100644 --- a/backend/app/routers/projects.py +++ b/backend/app/routers/projects.py @@ -50,8 +50,7 @@ async def save_project( es: Elasticsearch = Depends(dependencies.get_elasticsearchclient), ): project = ProjectDB( - **project_in.dict(), - creator=user, + **project_in.dict() ) await project.insert() @@ -220,9 +219,9 @@ async def get_projects( enable_admin: bool = False, ): # TODO check if the current user is a member OR creator - query = ProjectDB.creator.email == user_id - - projects_and_count = await ProjectDB.find(*query).to_list() + projects_and_count = await ProjectDB.aggregate( + [_get_page_query(skip, limit, sort_field="email", ascending=True)], + ).to_list() page_metadata = _construct_page_metadata(projects_and_count, skip, limit) # TODO have to change _id this way otherwise it won't work From 1c4d005c769ad09d7df44df50f60154f2eec9cc8 Mon Sep 17 00:00:00 2001 From: toddn Date: Tue, 30 Jul 2024 10:17:58 -0500 Subject: [PATCH 19/28] get project works --- .run/Python tests in tests.run.xml | 7 ++++--- backend/app/routers/projects.py | 11 ++++++----- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.run/Python tests in tests.run.xml b/.run/Python tests in tests.run.xml index 2fd8345de..2f33fbef9 100644 --- a/.run/Python tests in tests.run.xml +++ b/.run/Python tests in tests.run.xml @@ -3,9 +3,10 @@