File size: 13,765 Bytes
dd4ae81 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 |
{
"AstraDB": {
"destination_connector": "AstraDB",
"imports": "from unstructured_ingest.v2.processes.connectors.astradb import (AstraDBConnectionConfig, AstraDBAccessConfig, AstraDBUploadStagerConfig, AstraDBUploaderConfig)",
"configs": "destination_connection_config=AstraDBConnectionConfig(\n access_config=AstraDBAccessConfig(\n api_endpoint=os.getenv(\"ASTRA_DB_API_ENDPOINT\"),\n token=os.getenv(\"ASTRA_DB_APPLICATION_TOKEN\")\n )\n),\nstager_config=AstraDBUploadStagerConfig(),\nuploader_config=AstraDBUploaderConfig(\n namespace=os.getenv(\"ASTRA_DB_NAMESPACE\"),\n collection_name=os.getenv(\"ASTRA_DB_COLLECTION\"),\n embedding_dimension=os.getenv(\"ASTRA_DB_EMBEDDING_DIMENSIONS\")\n)",
"docs": "https://docs.unstructured.io/api-reference/ingest/destination-connector/astradb"
},
"AzureBlobStorage": {
"destination_connector": "AzureBlobStorage",
"imports": "from unstructured_ingest.v2.processes.connectors.fsspec.azure import (AzureConnectionConfig, AzureAccessConfig, AzureUploaderConfig)",
"configs": "destination_connection_config=AzureConnectionConfig(\n access_config=AzureAccessConfig(\n account_name=os.getenv(\"AZURE_STORAGE_ACCOUNT_NAME\"),\n account_key=os.getenv(\"AZURE_STORAGE_ACCOUNT_KEY\")\n )\n),\nuploader_config=AzureUploaderConfig(remote_url=os.getenv(\"AZURE_STORAGE_ACCOUNT_URL\"))",
"docs": "https://docs.unstructured.io/api-reference/ingest/destination-connector/azure"
},
"Box": {
"destination_connector": "Box",
"imports": "from unstructured_ingest.v2.processes.connectors.fsspec.box import (BoxAccessConfig, BoxConnectionConfig, BoxIndexerConfig, BoxUploaderConfig)",
"configs": "destination_connection_config=BoxConnectionConfig(\n access_config=BoxAccessConfig(\n box_app_config=os.getenv(\"BOX_APP_CONFIG_PATH\")\n )\n),\nuploader_config=BoxUploaderConfig(\n remote_url=os.getenv(\"BOX_REMOTE_URL\")\n)",
"docs": "https://docs.unstructured.io/api-reference/ingest/destination-connector/box"
},
"Couchbase": {
"destination_connector": "Couchbase",
"imports": "from unstructured_ingest.v2.processes.connectors.couchbase import (CouchbaseAccessConfig, CouchbaseConnectionConfig, CouchbaseUploadStagerConfig, CouchbaseUploaderConfig)",
"configs": "destination_connection_config=CouchbaseConnectionConfig(\n access_config=CouchbaseAccessConfig(\n password=os.getenv(\"CB_PASSWORD\"),\n ),\n connection_string=os.getenv(\"CB_CONN_STR\"),\n username=os.getenv(\"CB_USERNAME\"),\n bucket=os.getenv(\"CB_BUCKET\"),\n scope=os.getenv(\"CB_SCOPE\"),\n collection=os.getenv(\"CB_COLLECTION\")\n),\nstager_config=CouchbaseUploadStagerConfig(),\nuploader_config=CouchbaseUploaderConfig(batch_size=100)",
"docs": "https://docs.unstructured.io/api-reference/ingest/destination-connector/couchbase"
},
"Databricks Volumes": {
"destination_connector": "Databricks Volumes",
"imports": "from unstructured_ingest.v2.processes.connectors.databricks_volumes import (DatabricksVolumesConnectionConfig, DatabricksVolumesAccessConfig, DatabricksVolumesUploaderConfig)",
"configs": "destination_connection_config=DatabricksVolumesConnectionConfig(\n access_config=DatabricksVolumesAccessConfig(\n username=os.getenv(\"DATABRICKS_USERNAME\"),\n password=os.getenv(\"DATABRICKS_PASSWORD\")\n ),\n host=os.getenv(\"DATABRICKS_HOST\")\n),\nuploader_config=DatabricksVolumesUploaderConfig(\n catalog=os.getenv(\"DATABRICKS_CATALOG\"),\n volume=os.getenv(\"DATABRICKS_VOLUME\")\n)",
"docs": "https://docs.unstructured.io/api-reference/ingest/destination-connector/databricks-volumes"
},
"Dropbox": {
"destination_connector": "Dropbox",
"imports": "from unstructured_ingest.v2.processes.connectors.fsspec.dropbox import (DropboxAccessConfig, DropboxConnectionConfig, DropboxUploaderConfig)",
"configs": "destination_connection_config=DropboxConnectionConfig(\n access_config=DropboxAccessConfig(\n token=os.getenv(\"DROPBOX_ACCESS_TOKEN\")\n )\n),\nuploader_config=DropboxUploaderConfig(\n remote_url=os.getenv(\"DROPBOX_REMOTE_URL\")\n)",
"docs": "https://docs.unstructured.io/api-reference/ingest/destination-connector/dropbox"
},
"Elasticsearch": {
"destination_connector": "Elasticsearch",
"imports": "from unstructured_ingest.v2.processes.connectors.elasticsearch import (ElasticsearchConnectionConfig, ElasticsearchAccessConfig, ElasticsearchUploadStagerConfig, ElasticsearchUploaderConfig)",
"configs": "destination_connection_config=ElasticsearchConnectionConfig(\n access_config=ElasticsearchAccessConfig(\n password=os.getenv(\"ELASTICSEARCH_PASSWORD\"),\n ssl_assert_fingerprint=os.getenv(\"ELASTICSEARCH_SSL_ASSERT_FINGERPRINT\")\n ),\n hosts=[os.getenv(\"ELASTICSEARCH_HOST\")],\n username=os.getenv(\"ELASTICSEARCH_USERNAME\"),\n ca_certs=os.getenv(\"ELASTICSEARCH_CA_CERTS\")\n),\nstager_config=ElasticsearchUploadStagerConfig(index_name=os.getenv(\"ELASTICSEARCH_INDEX_NAME\")),\nuploader_config=ElasticsearchUploaderConfig(index_name=os.getenv(\"ELASTICSEARCH_INDEX_NAME\"))",
"docs": "https://docs.unstructured.io/api-reference/ingest/destination-connector/elasticsearch"
},
"Google Cloud Storage": {
"destination_connector": "Google Cloud Storage",
"imports": "from unstructured_ingest.v2.processes.connectors.fsspec.gcs import (GcsConnectionConfig, GcsAccessConfig, GcsUploaderConfig)",
"configs": "destination_connection_config=GcsConnectionConfig(\n access_config=GcsAccessConfig(\n service_account_key=os.getenv(\"GCS_SERVICE_ACCOUNT_KEY\")\n )\n),\nuploader_config=GcsUploaderConfig(remote_url=os.getenv(\"GCS_REMOTE_URL\"))",
"docs": "https://docs.unstructured.io/api-reference/ingest/destination-connector/google-cloud-service"
},
"KDB": {
"destination_connector": "KDB",
"imports": "from unstructured_ingest.v2.processes.connectors.kdbai import (KdbaiConnectionConfig, KdbaiAccessConfig, KdbaiUploadStagerConfig, KdbaiUploaderConfig)",
"configs": "destination_connection_config=KdbaiConnectionConfig(\n access_config=KdbaiAccessConfig(\n api_key=os.getenv(\"KDBAI_API_KEY\")\n ),\n endpoint=os.getenv(\"KDBAI_ENDPOINT\")\n),\nstager_config=KdbaiUploadStagerConfig(),\nuploader_config=KdbaiUploaderConfig(table_name=os.getenv(\"KDBAI_TABLE\"))",
"docs": "https://docs.unstructured.io/api-reference/ingest/destination-connector/kdbai"
},
"Local directory": {
"destination_connector": "Local directory",
"imports": "from unstructured_ingest.v2.processes.connectors.local import LocalUploaderConfig",
"configs": "uploader_config=LocalUploaderConfig(output_dir=os.getenv(\"LOCAL_FILE_OUTPUT_DIR\"))",
"docs": "https://docs.unstructured.io/api-reference/ingest/destination-connector/local"
},
"Milvus": {
"destination_connector": "Milvus",
"imports": "from unstructured_ingest.v2.processes.connectors.milvus import (MilvusConnectionConfig, MilvusAccessConfig, MilvusUploadStagerConfig, MilvusUploaderConfig)",
"configs": "destination_connection_config=MilvusConnectionConfig(\n access_config=MilvusAccessConfig(\n password=os.getenv(\"MILVUS_PASSWORD\")\n ),\n uri=os.getenv(\"MILVUS_URI\"),\n user=os.getenv(\"MILVUS_USER\"),\n db_name=os.getenv(\"MILVUS_DB\")\n),\nstager_config=MilvusUploadStagerConfig(),\nuploader_config=MilvusUploaderConfig(collection_name=os.getenv(\"MILVUS_COLLECTION\"))",
"docs": "https://docs.unstructured.io/api-reference/ingest/destination-connector/milvus"
},
"MongoDB": {
"destination_connector": "MongoDB",
"imports": "from unstructured_ingest.v2.processes.connectors.mongodb import (MongoDBAccessConfig, MongoDBConnectionConfig, MongoDBUploadStagerConfig, MongoDBUploaderConfig)",
"configs": "destination_connection_config=MongoDBConnectionConfig(\n access_config=MongoDBAccessConfig(\n uri=os.getenv(\"MONGODB_URI\")\n ),\n database=os.getenv(\"MONGODB_DATABASE\"),\n collection=os.getenv(\"MONGODB_COLLECTION\")\n),\nstager_config=MongoDBUploadStagerConfig(),\nuploader_config=MongoDBUploaderConfig(batch_size=100)",
"docs": "https://docs.unstructured.io/api-reference/ingest/destination-connector/mongodb"
},
"OpenSearch": {
"destination_connector": "OpenSearch",
"imports": "from unstructured_ingest.v2.processes.connectors.opensearch import (OpenSearchConnectionConfig, OpenSearchAccessConfig, OpensearchUploadStagerConfig, OpensearchUploaderConfig)",
"configs": "destination_connection_config=OpenSearchConnectionConfig(\n access_config=OpenSearchAccessConfig(\n password=os.getenv(\"OPENSEARCH_PASSWORD\"),\n use_ssl=True\n ),\n hosts=[os.getenv(\"OPENSEARCH_HOST\")],\n username=os.getenv(\"OPENSEARCH_USERNAME\")\n),\nstager_config=OpensearchUploadStagerConfig(\n index_name=os.getenv(\"OPENSEARCH_INDEX_NAME\")\n),\nuploader_config=OpensearchUploaderConfig(\n index_name=os.getenv(\"OPENSEARCH_INDEX_NAME\")\n)",
"docs": "https://docs.unstructured.io/api-reference/ingest/destination-connector/opensearch"
},
"Pinecone": {
"destination_connector": "Pinecone",
"imports": "from unstructured_ingest.v2.processes.connectors.pinecone import (PineconeConnectionConfig, PineconeAccessConfig, PineconeUploaderConfig, PineconeUploadStagerConfig)",
"configs": "destination_connection_config=PineconeConnectionConfig(\n access_config=PineconeAccessConfig(\n api_key=os.getenv(\"PINECONE_API_KEY\")\n ),\n index_name=os.getenv(\"PINECONE_INDEX_NAME\")\n),\nstager_config=PineconeUploadStagerConfig(),\nuploader_config=PineconeUploaderConfig()",
"docs": "https://docs.unstructured.io/api-reference/ingest/destination-connector/pinecone"
},
"S3": {
"destination_connector": "S3",
"imports": "from unstructured_ingest.v2.processes.connectors.fsspec.s3 import (S3ConnectionConfig, S3AccessConfig, S3UploaderConfig)",
"configs": "destination_connection_config=S3ConnectionConfig(\n access_config=S3AccessConfig(\n key=os.getenv(\"AWS_ACCESS_KEY_ID\"),\n secret=os.getenv(\"AWS_SECRET_ACCESS_KEY\")\n )\n),\nuploader_config=S3UploaderConfig(remote_url=os.getenv(\"AWS_S3_URL\"))",
"docs": "https://docs.unstructured.io/api-reference/ingest/destination-connector/s3"
},
"SFTP": {
"destination_connector": "SFTP",
"imports": "from unstructured_ingest.v2.processes.connectors.fsspec.sftp import (SftpConnectionConfig, SftpAccessConfig, SftpUploaderConfig)",
"configs": "destination_connection_config=SftpConnectionConfig(\n access_config=SftpAccessConfig(password=os.getenv(\"SFTP_PASSWORD\")),\n host=os.getenv(\"SFTP_HOST\"),\n port=os.getenv(\"SFTP_PORT\"),\n username=os.getenv(\"SFTP_USERNAME\")\n),\nuploader_config=SftpUploaderConfig(remote_url=os.getenv(\"SFTP_REMOTE_URL\"))",
"docs": "https://docs.unstructured.io/api-reference/ingest/destination-connector/sftp"
},
"SingleStore": {
"destination_connector": "SingleStore",
"imports": "from unstructured_ingest.v2.processes.connectors.singlestore import (SingleStoreConnectionConfig, SingleStoreAccessConfig, SingleStoreUploadStagerConfig, SingleStoreUploaderConfig)",
"configs": "destination_connection_config=SingleStoreConnectionConfig(\n host=os.getenv(\"SINGLESTORE_HOST\"),\n port=os.getenv(\"SINGLESTORE_PORT\"),\n user=os.getenv(\"SINGLESTORE_USER\"),\n database=os.getenv(\"SINGLESTORE_DB\"),\n access_config=SingleStoreAccessConfig(\n password=os.getenv(\"SINGLESTORE_PASSWORD\")\n )\n),\nstager_config=SingleStoreUploadStagerConfig(drop_empty_cols=True),\nuploader_config=SingleStoreUploaderConfig(table_name=os.getenv(\"SINGLESTORE_TABLE\"))",
"docs": "https://docs.unstructured.io/api-reference/ingest/destination-connector/singlestore"
},
"SQL": {
"destination_connector": "SQL",
"imports": "from unstructured_ingest.v2.processes.connectors.sql import (SQLConnectionConfig, SQLAccessConfig, SQLUploaderConfig, SQLUploadStagerConfig)",
"configs": "destination_connection_config=SQLConnectionConfig(\n access_config=SQLAccessConfig(\n username=os.getenv(\"PGUSER\"),\n password=os.getenv(\"PGPASSWORD\")\n ),\n db_type=os.getenv(\"SQL_DB_TYPE\"),\n host=os.getenv(\"PGHOST\"),\n port=os.getenv(\"PGPORT\"),\n database=os.getenv(\"PGDATABASE\")\n),\nstager_config=SQLUploadStagerConfig(),\nuploader_config=SQLUploaderConfig()",
"docs": "https://docs.unstructured.io/api-reference/ingest/destination-connector/sql"
},
"Weaviate": {
"destination_connector": "Weaviate",
"imports": "from unstructured_ingest.v2.processes.connectors.weaviate import (WeaviateConnectionConfig, WeaviateAccessConfig, WeaviateUploaderConfig, WeaviateUploadStagerConfig)",
"configs": "destination_connection_config=WeaviateConnectionConfig(\n access_config=WeaviateAccessConfig(\n api_key=os.getenv(\"WEAVIATE_API_KEY\")\n ),\n host_url=os.getenv(\"WEAVIATE_URL\"),\n class_name=os.getenv(\"WEAVIATE_COLLECTION_CLASS_NAME\")\n),\nstager_config=WeaviateUploadStagerConfig(),\nuploader_config=WeaviateUploaderConfig()",
"docs": "https://docs.unstructured.io/api-reference/ingest/destination-connector/weaviate"
}
}
|