Artifacts
# Get the latest lib from Rapidcanvas
# !pip install --extra-index-url=https://us-central1-python.pkg.dev/rapidcanvas-361003/pypi/simple utils==0.12dev0
from utils.rc.client.requests import Requests
from utils.rc.client.auth import AuthClient
from utils.rc.dtos.project import Project
from utils.rc.dtos.dataset import Dataset
from utils.rc.dtos.recipe import Recipe
from utils.rc.dtos.transform import Transform
from utils.rc.dtos.artifact import Artifact
from utils.rc.dtos.template_v2 import TemplateV2, TemplateTransformV2
import pandas as pd
import logging
from utils.utils.log_util import LogUtil
LogUtil.set_basic_config(format='%(levelname)s:%(message)s', level=logging.INFO)
# Requests.setRootHost("https://test.dev.rapidcanvas.net/api/")
AuthClient.setToken()
INFO:Authentication successful
Artifact.add_file("my-outside-artifact-v1", "data/titanic.csv")
Artifact.add_file("my-outside-artifact-v1", "data/titanic-v2.csv")
INFO:File successfully added
INFO:File successfully added
project = Project.create(
name="Example Artifact",
description="Testing python lib",
createEmpty=True
)
INFO:Found existing project by name: Example Artifact
INFO:Deleting existing project
INFO:Creating new project by name: Example Artifact
project.id
'4c4ab654-11fe-475a-a15b-6a2bb0819d2f'
artifacts = Artifact.get_all()
if "my-artifact-v1" in artifacts:
Artifact.delete("my-artifact-v1")
if "my-artifact-v2" in artifacts:
Artifact.delete("my-artifact-v2")
INFO:Artifact my-artifact-v2 deleted successfully!
titanic = project.addDataset(
dataset_name="titanic",
dataset_description="titanic golden",
dataset_file_path="data/titanic.csv"
)
INFO:Dataset titanic not found!
INFO:Creating dataset titanic.
INFO:Uploading file data/titanic.csv ...
INFO:Processing files...
INFO:Done!
recipe = project.addRecipe([titanic], name="recipe_2", artifacts=["my-outside-artifact-v1"])
INFO:Creating new recipe
template = TemplateV2(
name="CreateArtifact", description="CreateArtifact", project_id=project.id, source="CUSTOM", status="ACTIVE", tags=["Number", "datatype-long"]
)
template_transform = TemplateTransformV2(type = "python", params=dict(notebookName="CreateArtifact.ipynb"))
template.base_transforms = [template_transform]
template.publish("transforms/CreateArtifact.ipynb")
INFO:Publishing template | data=TemplateV2(name='CreateArtifact', display_name=None, id=None, version='1.0', project_id='4c4ab654-11fe-475a-a15b-6a2bb0819d2f', projectId='4c4ab654-11fe-475a-a15b-6a2bb0819d2f', type=<TemplateType.NOTEBOOK: 'NOTEBOOK'>, code=None, description='CreateArtifact', tags=['Number', 'datatype-long'], secondary_tags=None, requirements='', baseTransforms=[TemplateTransformV2(type='python', params={'notebookName': 'CreateArtifact.ipynb'})], base_transforms=[TemplateTransformV2(type='python', params={'notebookName': 'CreateArtifact.ipynb'})], source='PROJECT', status='ACTIVE', category='BASIC', inputs=[]) INFO:Template Published 2023-09-13 11:25:03.516 INFO papermill: Executing notebook with kernel: python3 INFO:User authenticated successfully INFO:Creating template input | nb_stage=COMPILE_TIME INFO:Creating template input | nb_stage=COMPILE_TIME INFO:Creating template input | nb_stage=COMPILE_TIME INFO:Creating template input | nb_stage=COMPILE_TIME INFO: ***************************************************************************** ** CREATING INPUTS: artifactName, artifactName2, nRows, artifactFileName ** ***************************************************************************** INFO:Inputs created successfully | template_id=f2a2696f-64d9-448a-ac1e-06bd47408724 Inputs created successfully | template_id=f2a2696f-64d9-448a-ac1e-06bd47408724
transform = Transform()
transform.templateId = template.id
transform.name = "transform_1"
transform.variables = {
"artifactName": "my-artifact-v1",
"artifactName2": "my-artifact-v2",
"nRows": "60",
"artifactFileName": "sixty"
}
recipe.prepareForLocal(transform, contextId="artifact-recipe")
INFO:os.path.isdir(basePath) - True
INFO:basePath - /Users/nikunj/Documents/rapidcanvas/rc-core/dataapp/data//temp/artifact-recipe
INFO:Downloading datasets, artifacts, models ...
INFO:Downloading dataset titanic ...
INFO:Checking stats for titanic
INFO:Downloading dataset schema for titanic ...
INFO:Downloading artifact my-outside-artifact-v1 ...
INFO:Artifact file downloaded successfully! to /Users/nikunj/Documents/rapidcanvas/rc-core/dataapp/data/tenant-test-tenant/artifacts/my-outside-artifact-v1/titanic-v2.csv
INFO:Artifact file downloaded successfully! to /Users/nikunj/Documents/rapidcanvas/rc-core/dataapp/data/tenant-test-tenant/artifacts/my-outside-artifact-v1/titanic.csv
INFO:Prepared to run locally. Use artifact-recipe in your notebook file as context id
'artifact-recipe'
recipe.add_transform(transform)
INFO:Adding new transform
INFO:Transform added Successfully
recipe.run()
INFO:Started running
INFO:You can look at the progress on UI at https://test.dev.rapidcanvas.net/#/projects/4c4ab654-11fe-475a-a15b-6a2bb0819d2f
INFO:No errors and warnings found
recipe.getChildrenDatasets()
{'titanic_out_v2': Dataset(id='34e3b55c-7361-4be0-977f-b9edbcd9c434', name='titanic_out_v2', project_id=None, display_name='titanic_out_v2', description='titanic_out_v2', file_path=None, status=<DatasetStatus.BUILT: 'BUILT'>, data_source_id=None, data_source_options=None, datasetMeta=None, entityMeta=DatasetMeta(entity_type='BASE', entity_ontology='NONE', entity_view_type=None, encoding=None, separator=None, header=None, data_type_map=None, ontology_map=None), dataset_meta=DatasetMeta(entity_type='BASE', entity_ontology='NONE', entity_view_type='NONE', encoding='', separator='', header='', data_type_map={}, ontology_map={}), metadata={})}
all_artifacts = Artifact.get_all()
all_artifacts
['image_models',
'my-artifact-v1',
'my-artifact-v2',
'my-outside-artifact-v1',
'test-artifact']
assert 'my-artifact-v1' in all_artifacts, "artifacts dont match"
artifact = Artifact.get('my-artifact-v1')
assert 'sixty.csv' in artifact.files, "artifact files dont match"
assert 'my-artifact-v2' in all_artifacts, "artifacts dont match"
artifact = Artifact.get('my-artifact-v2')
assert 'sixty.csv' in artifact.files, "artifact files dont match"
Creating artifact outside of recipe
all_artifacts = Artifact.get_all()
assert 'my-outside-artifact-v1' in all_artifacts, "artifacts dont match"
artifact = Artifact.get('my-outside-artifact-v1')
assert 'titanic.csv' in artifact.files, "artifact files dont match"
assert 'titanic-v2.csv' in artifact.files, "artifact files dont match"
Artifact.delete_file("my-outside-artifact-v1", "titanic-v2.csv")
INFO:File successfully deleted
artifact = Artifact.get('my-outside-artifact-v1')
assert 'titanic-v2.csv' not in artifact.files, "artifact files dont match"
Artifact.add_file("my-outside-artifact-v2", "data/titanic.csv")
Artifact.add_file("my-outside-artifact-v2", "data/titanic-v2.csv")
INFO:File successfully added
INFO:File successfully added
Artifact.delete('my-outside-artifact-v2')
INFO:Artifact my-outside-artifact-v2 deleted successfully!
all_artifacts = Artifact.get_all()
assert 'my-outside-artifact-v2' not in all_artifacts, "artifacts dont match"