'use client' import type { FC } from 'react' import React, { useCallback, useEffect, useState } from 'react' import { useTranslation } from 'react-i18next' import s from './index.module.css' import NoData from './no-data' import Firecrawl from './firecrawl' import JinaReader from './jina-reader' import cn from '@/utils/classnames' import { useModalContext } from '@/context/modal-context' import type { CrawlOptions, CrawlResultItem } from '@/models/datasets' import { fetchDataSources } from '@/service/datasets' import { type DataSourceItem, DataSourceProvider } from '@/models/common' type Props = { onPreview: (payload: CrawlResultItem) => void checkedCrawlResult: CrawlResultItem[] onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void onCrawlProviderChange: (provider: DataSourceProvider) => void onJobIdChange: (jobId: string) => void crawlOptions: CrawlOptions onCrawlOptionsChange: (payload: CrawlOptions) => void } const Website: FC = ({ onPreview, checkedCrawlResult, onCheckedCrawlResultChange, onCrawlProviderChange, onJobIdChange, crawlOptions, onCrawlOptionsChange, }) => { const { t } = useTranslation() const { setShowAccountSettingModal } = useModalContext() const [isLoaded, setIsLoaded] = useState(false) const [selectedProvider, setSelectedProvider] = useState(DataSourceProvider.jinaReader) const [sources, setSources] = useState([]) useEffect(() => { onCrawlProviderChange(selectedProvider) }, [selectedProvider, onCrawlProviderChange]) const checkSetApiKey = useCallback(async () => { const res = await fetchDataSources() as any setSources(res.sources) // If users have configured one of the providers, select it. const availableProviders = res.sources.filter((item: DataSourceItem) => [DataSourceProvider.jinaReader, DataSourceProvider.fireCrawl].includes(item.provider), ) if (availableProviders.length > 0) setSelectedProvider(availableProviders[0].provider) }, []) useEffect(() => { checkSetApiKey().then(() => { setIsLoaded(true) }) // eslint-disable-next-line react-hooks/exhaustive-deps }, []) const handleOnConfig = useCallback(() => { setShowAccountSettingModal({ payload: 'data-source', onCancelCallback: checkSetApiKey, }) }, [checkSetApiKey, setShowAccountSettingModal]) if (!isLoaded) return null return (
{t('datasetCreation.stepOne.website.chooseProvider')}
{ selectedProvider === DataSourceProvider.fireCrawl ? sources.find(source => source.provider === DataSourceProvider.fireCrawl) ? ( ) : ( ) : sources.find(source => source.provider === DataSourceProvider.jinaReader) ? ( ) : ( ) }
) } export default React.memo(Website)