add Set up source button for optional adapters and fix cache headers

apps/web/app.js

@@ -278,13 +278,21 @@
   const sourceText = (source, keys, fallback='Not returned') => { for (const key of keys) if (source?.[key] !== undefined && source[key] !== null && source[key] !== '') return source[key]; return fallback; };
   function sourceMessage(text, error = false) { const el = $('sourcesMessage'); if (el) { el.textContent = text || ''; el.className = `sources-message${error ? ' error' : ''}`; } }
   function renderSourceSelect(){const multi=$('directDiscoverySources'),enabled=sources.filter(sourceState);if(multi)multi.innerHTML=enabled.map(s=>`<option value="${esc(s.id)}">${esc(sourceLabel(s))}</option>`).join('')||'<option value="" disabled>No enabled sources are available</option>';const help=$('directDiscoverySourceHelp');if(help)help.textContent=enabled.length?'Select the approved sources to use for this run.':'No enabled sources are available. Configure, test, and enable a source first.';}
-  function renderSources(){renderSourceSelect();const list=$('sourcesList');if(!sources.length){list.innerHTML='<div class="source-empty">No registered sources returned by the workspace. Save configuration to add one.</div>';return;}list.innerHTML=sources.map(source=>{const config=sourceJson(source.config_json||source.config),policy=sourceJson(source.policy_json||source.policy),quota=sourceJson(source.quota_json||source.quota),owner=sourceText(source,['owner'],policy.owner||config.owner||'Not assigned'),terms=sourceText(source,['terms_status'],policy.terms_status||config.terms_status||'Pending'),rate=sourceText(source,['rate_limit'],policy.rate_limit||config.rate_limit||'Not set'),daily=sourceText(source,['daily_quota'],quota.daily_quota||config.daily_quota||'Not set'),health=sourceText(source,['health_status','health'],'Not tested'),ready=source.id&&terms==='approved'&&owner!=='Not assigned'&&rate!=='Not set'&&daily!=='Not set'&&/healthy|passed|success/i.test(health);return `<article class="source-row" data-source-id="${esc(source.id||'')}"><div class="source-row-main"><div class="source-title-line"><strong>${esc(sourceLabel(source))}</strong><span class="source-type">${esc(sourceType(source))}</span><span class="source-status ${sourceStatus(source)}">${esc(sourceStatus(source))}</span></div><small>Registry configuration · no discovery criteria stored</small></div><dl class="source-facts source-registry-facts"><div><dt>Owner</dt><dd>${esc(owner)}</dd></div><div><dt>Terms</dt><dd>${esc(terms)}</dd></div><div><dt>Rate limit</dt><dd>${esc(rate)}</dd></div><div><dt>Daily quota</dt><dd>${esc(daily)}</dd></div><div><dt>Health</dt><dd>${esc(health)}</dd></div><div><dt>Status</dt><dd>${esc(sourceStatus(source))}</dd></div></dl><div class="source-actions"><button class="button ghost compact" type="button" data-source-action="test" data-source-id="${esc(source.id||'')}" ${source.id?'':'disabled title="Save configuration before testing."'}>Test</button><button class="button ghost compact" type="button" data-source-action="toggle" data-source-id="${esc(source.id||'')}" ${ready?'':'disabled title="Enable is unavailable until terms, owner, limits, and a successful test are recorded."'}>${sourceState(source)?'Disable':'Enable'}</button></div></article>`;}).join('');}
+  function renderSources(){renderSourceSelect();const list=$('sourcesList');if(!sources.length){list.innerHTML='<div class="source-empty">No registered sources returned by the workspace. Save configuration to add one.</div>';return;}list.innerHTML=sources.map(source=>{const config=sourceJson(source.config_json||source.config),policy=sourceJson(source.policy_json||source.policy),quota=sourceJson(source.quota_json||source.quota),owner=sourceText(source,['owner'],policy.owner||config.owner||'Not assigned'),terms=sourceText(source,['terms_status'],policy.terms_status||config.terms_status||'Pending'),rate=sourceText(source,['rate_limit'],policy.rate_limit||config.rate_limit||'Not set'),daily=sourceText(source,['daily_quota'],quota.daily_quota||config.daily_quota||'Not set'),health=sourceText(source,['health_status','health'],'Not tested'),ready=source.id&&terms==='approved'&&owner!=='Not assigned'&&rate!=='Not set'&&daily!=='Not set'&&/healthy|passed|success/i.test(health);const actions=source.optional?'<div class="source-actions"><button class="button primary compact" type="button" data-source-action="setup" data-source-code="'+esc(sourceType(source))+'">Set up source</button></div>':'<div class="source-actions"><button class="button ghost compact" type="button" data-source-action="test" data-source-id="'+esc(source.id||'')+'">Test</button><button class="button ghost compact" type="button" data-source-action="toggle" data-source-id="'+esc(source.id||'')+'" '+(ready?'':'disabled title="Enable is unavailable until terms, owner, limits, and a successful test are recorded."')+'>'+(sourceState(source)?'Disable':'Enable')+'</button></div>';return '<article class="source-row" data-source-id="'+esc(source.id||'')+'"><div class="source-row-main"><div class="source-title-line"><strong>'+esc(sourceLabel(source))+'</strong><span class="source-type">'+esc(sourceType(source))+'</span><span class="source-status '+esc(sourceStatus(source))+'">'+esc(sourceStatus(source))+'</span></div><small>Registry configuration · no discovery criteria stored</small></div><dl class="source-facts source-registry-facts"><div><dt>Owner</dt><dd>'+esc(owner)+'</dd></div><div><dt>Terms</dt><dd>'+esc(terms)+'</dd></div><div><dt>Rate limit</dt><dd>'+esc(rate)+'</dd></div><div><dt>Daily quota</dt><dd>'+esc(daily)+'</dd></div><div><dt>Health</dt><dd>'+esc(health)+'</dd></div><div><dt>Status</dt><dd>'+esc(sourceStatus(source))+'</dd></div></dl>'+actions+'</article>';}).join('');}
   function renderSourceRecords(items){const list=$('sourceRecordsList');list.innerHTML=items.length?`<div class="source-record-table"><table><tbody>${items.slice(0,25).map(record=>`<tr><td>${esc(record.name||record.title||record.id||'Unnamed record')}</td><td>${esc(record.source_name||record.source||'Unknown source')}</td><td>${esc(record.status||'Pending')}</td></tr>`).join('')}</tbody></table></div>`:'<div class="source-empty">No source records returned by the workspace.</div>';}
   async function loadSources(){sourceMessage('Loading sources…');try{const [sourcePayload,adapterPayload,recordPayload]=await Promise.all([jsonRequest('/api/v1/sources'),jsonRequest('/api/v1/sources/adapters'),jsonRequest('/api/v1/source-records?page_size=25')]);const configured=sourceItems(sourcePayload),registeredCodes=new Set(configured.map(sourceType)),optional=sourceItems(adapterPayload).filter(a=>!registeredCodes.has(a.source_code)).map(a=>({...a,kind:a.source_code,optional:true,configured:false,available:Boolean(a.available),enabled:false}));sources=[...configured,...optional];renderSources();renderSourceRecords(sourceItems(recordPayload));$('sourcesUpdatedAt').textContent=`Updated ${new Date().toLocaleTimeString([], {hour:'2-digit',minute:'2-digit'})}`;sourceMessage(sources.some(sourceState)?'':'No enabled source. Configure, test, and enable a source before discovery.');}catch(error){sources=[];renderSources();renderSourceRecords([]);if(error.message!=='unauthorized')sourceMessage(error.message||'Unable to load source registry.',true);}}
   function setSourceButtons(busy,status=''){['sourceSaveBtn','sourceTestBtn','sourceEnableBtn'].forEach(id=>{const el=$(id);if(el)el.disabled=busy||(id!=='sourceSaveBtn'&&!selectedSourceId);});if(status)$('sourceStatus').textContent=status;}
   async function saveSource(event){event.preventDefault();const form=event.currentTarget,fields=Object.fromEntries(new FormData(form).entries());if(fields.source_type==='csv'&&!fields.csv_content.trim()){message('sourceFormMessage','CSV content is required for a CSV source.',true);return;}let providerSettings={};try{providerSettings=fields.provider_settings.trim()?JSON.parse(fields.provider_settings):{};}catch{message('sourceFormMessage','Provider settings must be valid JSON.',true);return;}setSourceButtons(true,'Saving source configuration…');message('sourceFormMessage','Saving source configuration…');const config={owner:fields.owner,terms_url:fields.terms_url,terms_status:fields.terms_status,rate_limit:fields.rate_limit,daily_quota:Number(fields.daily_quota),provider_settings:fields.provider_settings};if(fields.source_type==='csv')config.csv=fields.csv_content;else config.rows=[];try{const saved=await jsonRequest('/api/v1/sources',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({name:fields.name,kind:fields.source_type,config,policy:{owner:fields.owner,terms_url:fields.terms_url,terms_status:fields.terms_status,rate_limit:fields.rate_limit},quota:{daily_quota:Number(fields.daily_quota)},enabled:false})});selectedSourceId=saved.id||saved.source?.id||null;message('sourceFormMessage','Source configuration saved. It remains disabled until a successful test and explicit enablement.');await loadSources();}catch(error){if(error.message!=='unauthorized')message('sourceFormMessage',error.message||'Unable to save source configuration.',true);}finally{setSourceButtons(false);}}
   function findRegisteredSource(id){return sources.find(item=>String(item.id)===String(id)&&!item.optional)||sources.find(item=>String(item.id)===String(id));}
   async function sourceAction(id, action) {
+    if (action === 'setup') {
+      const code = id;
+      $('sourceName').value = code.replace(/_/g, ' ').replace(/\b\w/g, c => c.toUpperCase());
+      $('sourceType').value = 'provider';
+      $('sourceProviderSettings').value = JSON.stringify({provider: code});
+      document.getElementById('sources').scrollIntoView({behavior: 'smooth'});
+      return;
+    }
     const source = findRegisteredSource(id || selectedSourceId);
     if (!source?.id) { sourceMessage('Save source configuration before testing or enabling.', true); return; }
     try {
@@ -385,7 +393,7 @@ async function discoveryAction(action,run){if(!run?.id)return;const path=action=
   document.addEventListener('submit',e=>{if(e.target.id==='contactForm')saveContact(e.target);if(e.target.id==='noteForm')saveNote(e.target);if(e.target.id==='pipelineForm')saveStage(e.target);});
   document.addEventListener('click',e=>{if(e.target.id==='verifyBtn')verify();if(e.target.id==='recalculateScoreBtn')recalculateScore();if(e.target.id==='generateAiSuggestionBtn')generateAiSuggestion();if(e.target.id==='createOutreachDraftBtn')createOutreachDraft();const outreachButton=e.target.closest?.('[data-outreach-action]');if(outreachButton)approveOutreachDraft(outreachButton.dataset.outreachId);const aiButton=e.target.closest?.('[data-ai-action]');if(aiButton)aiAction(aiButton.dataset.aiAction,aiButton.dataset.aiId);const scoreEdit=e.target.closest?.('[data-score-edit]');if(scoreEdit)updateScoreRule(scoreEdit.dataset.scoreEdit,scoreEdit.dataset.scorePoints);if(e.target.id==='extractContactsBtn'&&selectedId)loadContactExtraction(selectedId,{extract:true});if(e.target.id==='refreshContactExtractionBtn'&&selectedId)loadContactExtraction(selectedId,{extract:true});if(e.target.id==='retryContactExtractionBtn'&&selectedId)loadContactExtraction(selectedId);if(e.target.id==='scanWebsiteBtn'&&selectedId)loadWebsiteScan(selectedId,{scan:true});if(e.target.id==='refreshWebsiteScanBtn'&&selectedId)loadWebsiteScan(selectedId);if(e.target.id==='retryWebsiteScanBtn'&&selectedId)loadWebsiteScan(selectedId);if(e.target.id==='runDomainCheckBtn')runDomainCheck();if(e.target.id==='retryDomainBtn'&&selectedId)loadDomainIntelligence(selectedId);const availability=e.target.closest?.('[data-domain-availability]');if(availability)checkDomainAvailability(availability.dataset.domain,availability);if(e.target.id==='retryDetailBtn'&&selectedId)loadDetail(selectedId);if(e.target.id==='retryDedupBtn'&&selectedId)loadMatchSuggestions(selectedId);if(e.target.id==='retryHistoryBtn'&&selectedId)loadMergeHistory(selectedId);if(e.target.id==='cancelMergeBtn'||e.target.id==='cancelMergeBtnSecondary')closeMergeDialog();if(e.target.id==='confirmMergeBtn')confirmMerge();const mergeButton=e.target.closest?.('[data-merge-target]');if(mergeButton)openMergeDialog(mergeButton.dataset.mergeTarget,mergeButton.dataset.mergeTargetName);const reverseButton=e.target.closest?.('[data-reverse-merge]');if(reverseButton)reverseMerge(reverseButton.dataset.reverseMerge);if(e.target.id==='retryJobDetailBtn'&&selectedJobId)loadJobDetail(selectedJobId);if(e.target.id==='cancelJobBtn')jobAction('cancel');if(e.target.id==='retryJobBtn')jobAction('retry');const row=e.target.closest?.('[data-job-id]');if(row)loadJobDetail(row.dataset.jobId);});
   $('loginForm').addEventListener('submit',login);$('logoutBtn').addEventListener('click',logout);$('directDiscoveryForm').addEventListener('submit',submitDirectDiscovery);$('directDiscoveryRefreshBtn').addEventListener('click',loadDiscoveryRuns);$('directDiscoveryRunsState').addEventListener('click',e=>{const row=e.target.closest?.('[data-discovery-run-id]');if(row){const run=discoveryRuns.find(item=>String(item.id)===String(row.dataset.discoveryRunId));if(run)loadDiscoveryRun(run);}if(e.target.id==='retryDiscoveryRunsBtn')loadDiscoveryRuns();});
-  $('directDiscoveryResultState').addEventListener('click',e=>{const aiButton=e.target.closest?.('[data-ai-assess]');if(aiButton){const bid=aiButton.dataset.aiAssess;const run=discoveryRuns.find(item=>String(item.id)===String(selectedDiscoveryRunId));const candidates=discoveryCandidates(run);const item=candidates.find(c=>String(c.id||c.business_id)===String(bid));assessCandidate(bid,item||{});}const button=e.target.closest?.('[data-run-action]');if(button){const run=discoveryRuns.find(item=>String(item.id)===String(selectedDiscoveryRunId));discoveryAction(button.dataset.runAction,run);}});$('searchInput').addEventListener('input',()=>{page=1;renderRows();});['scoreFilter','statusFilter','websiteClassFilter','pipelineFilter'].forEach(id=>$(id).addEventListener('change',()=>{page=1;loadData();}));$('pageSize').addEventListener('change',e=>{pageSize=Number(e.target.value);page=1;loadData();});$('nextPageBtn').addEventListener('click',()=>{if(hasNextPage){page+=1;loadData();}});$('refreshBtn').addEventListener('click',loadData);$('jobsRefreshBtn').addEventListener('click',()=>loadJobs());$('sourcesRefreshBtn').addEventListener('click',loadSources);$('sourceForm').addEventListener('submit',saveSource);$('sourceType').addEventListener('change',e=>{$('sourceCsvField').hidden=e.target.value!=='csv';});$('sourceTestBtn').addEventListener('click',()=>sourceAction(selectedSourceId,'test'));$('sourceEnableBtn').addEventListener('click',()=>sourceAction(selectedSourceId,'toggle'));$('sourcesList').addEventListener('click',e=>{const button=e.target.closest?.('[data-source-action]');if(button)sourceAction(button.dataset.sourceId,button.dataset.sourceAction);});$('addForm').addEventListener('submit',addProspect);$('csvInput').addEventListener('change',e=>{const file=e.target.files[0];if(file){const reader=new FileReader();reader.onload=()=>renderCsv(parseCsv(reader.result));reader.readAsText(file);}});$('menuBtn').addEventListener('click',()=>document.querySelector('.sidebar').classList.toggle('open'));document.querySelectorAll('[data-scroll]').forEach(b=>b.addEventListener('click',()=>document.querySelector(b.dataset.scroll)?.scrollIntoView()));
+  $('directDiscoveryResultState').addEventListener('click',e=>{const aiButton=e.target.closest?.('[data-ai-assess]');if(aiButton){const bid=aiButton.dataset.aiAssess;const run=discoveryRuns.find(item=>String(item.id)===String(selectedDiscoveryRunId));const candidates=discoveryCandidates(run);const item=candidates.find(c=>String(c.id||c.business_id)===String(bid));assessCandidate(bid,item||{});}const button=e.target.closest?.('[data-run-action]');if(button){const run=discoveryRuns.find(item=>String(item.id)===String(selectedDiscoveryRunId));discoveryAction(button.dataset.runAction,run);}});$('searchInput').addEventListener('input',()=>{page=1;renderRows();});['scoreFilter','statusFilter','websiteClassFilter','pipelineFilter'].forEach(id=>$(id).addEventListener('change',()=>{page=1;loadData();}));$('pageSize').addEventListener('change',e=>{pageSize=Number(e.target.value);page=1;loadData();});$('nextPageBtn').addEventListener('click',()=>{if(hasNextPage){page+=1;loadData();}});$('refreshBtn').addEventListener('click',loadData);$('jobsRefreshBtn').addEventListener('click',()=>loadJobs());$('sourcesRefreshBtn').addEventListener('click',loadSources);$('sourceForm').addEventListener('submit',saveSource);$('sourceType').addEventListener('change',e=>{$('sourceCsvField').hidden=e.target.value!=='csv';});$('sourceTestBtn').addEventListener('click',()=>sourceAction(selectedSourceId,'test'));$('sourceEnableBtn').addEventListener('click',()=>sourceAction(selectedSourceId,'toggle'));$('sourcesList').addEventListener('click',e=>{const button=e.target.closest?.('[data-source-action]');if(button){const code=button.dataset.sourceCode;if(code){sourceAction(code,button.dataset.sourceAction);}else{sourceAction(button.dataset.sourceId,button.dataset.sourceAction);}}});$('addForm').addEventListener('submit',addProspect);$('csvInput').addEventListener('change',e=>{const file=e.target.files[0];if(file){const reader=new FileReader();reader.onload=()=>renderCsv(parseCsv(reader.result));reader.readAsText(file);}});$('menuBtn').addEventListener('click',()=>document.querySelector('.sidebar').classList.toggle('open'));document.querySelectorAll('[data-scroll]').forEach(b=>b.addEventListener('click',()=>document.querySelector(b.dataset.scroll)?.scrollIntoView()));
   $('savedFilterForm').addEventListener('submit',saveCurrentFilters);$('savedFilterSelect').addEventListener('change',e=>{ $('deleteSavedFilterBtn').disabled=!e.target.value;if(e.target.value)loadSavedFilter(e.target.value);});$('deleteSavedFilterBtn').addEventListener('click',deleteSavedFilter);$('reviewQueueState').addEventListener('change',e=>{const input=e.target.closest?.('[data-review-id]');if(input){if(input.checked)selectedReviewIds.add(String(input.dataset.reviewId));else selectedReviewIds.delete(String(input.dataset.reviewId));updateBulkState();}});$('selectAllReview').addEventListener('change',e=>{reviewQueue.forEach(p=>e.target.checked?selectedReviewIds.add(String(p.id)):selectedReviewIds.delete(String(p.id)));renderReviewQueue({items:reviewQueue,count:$('reviewQueueCount').textContent});});$('bulkVerifyBtn').addEventListener('click',()=>bulkReview('verify'));$('bulkRejectBtn').addEventListener('click',()=>bulkReview('reject'));$('reviewQueueState').addEventListener('click',e=>{if(e.target.id==='retryReviewQueueBtn')loadReviewQueue();});document.querySelectorAll('[data-dashboard-filter]').forEach(card=>card.addEventListener('click',()=>{const kind=card.dataset.dashboardFilter;if(kind==='review'||kind==='suppressed')applyFilters({status:kind});else if(kind==='high')applyFilters({score:'high'});else if(kind==='fresh')applyFilters({});else applyFilters({status:'all',score:'all'});}));
   ['sourceFilter','geographyFilter','categoryFilter','contactStatusFilter','scoreFilter','statusFilter','websiteClassFilter','pipelineFilter'].forEach(id=>$(id)?.addEventListener('change',()=>{page=1;loadData();}));
   $('interactionForm').addEventListener('submit',saveInteraction);$('suppressionForm').addEventListener('submit',addSuppression);$('aiProviderForm').addEventListener('submit',saveAiProviderSettings);$('aiProvider').addEventListener('change',updateAiProviderForm);$('aiProviderRefreshBtn').addEventListener('click',loadAiProviderSettings);$('testAiProviderBtn').addEventListener('click',testAiProviderConnection);$('crmRefreshBtn').addEventListener('click',()=>{loadCrmPipeline();loadInteractions(selectedId);});$('reportsRefreshBtn').addEventListener('click',loadReports);$('suppressionRefreshBtn').addEventListener('click',loadSuppressions);$('pipelineViewToggle').addEventListener('click',()=>{crmListMode=!crmListMode;$('pipelineViewToggle').textContent=crmListMode?'▦ Board view':'☷ List view';$('pipelineViewToggle').setAttribute('aria-pressed',String(crmListMode));renderPipeline();});$('pipelineBoard').addEventListener('click',e=>{const selectButton=e.target.closest?.('[data-crm-select]');if(selectButton){selectedId=Number(selectButton.dataset.crmSelect);selectProspect(selectedId);loadInteractions(selectedId);$('crmActivity').scrollIntoView({behavior:'smooth',block:'start'});}const stageButton=e.target.closest?.('[data-crm-save-stage]');if(stageButton)saveCrmStage(stageButton.dataset.crmSaveStage);if(e.target.id==='retryCrmBtn')loadCrmPipeline();if(e.target.id==='retryInteractionsBtn'&&selectedId)loadInteractions(selectedId);});$('suppressionState').addEventListener('change',e=>{const input=e.target.closest?.('[data-suppression-id]');if(input){if(input.checked)selectedSuppressionIds.add(String(input.dataset.suppressionId));else selectedSuppressionIds.delete(String(input.dataset.suppressionId));updateSuppressionSelection();}});$('suppressionState').addEventListener('click',e=>{const button=e.target.closest?.('[data-remove-suppression]');if(button)removeSuppression(button.dataset.removeSuppression);if(e.target.id==='retrySuppressionsBtn')loadSuppressions();});$('selectAllSuppressions').addEventListener('change',e=>{suppressions.forEach(item=>e.target.checked?selectedSuppressionIds.add(String(item.id)):selectedSuppressionIds.delete(String(item.id)));renderSuppressions();});$('bulkReviewSuppressionsBtn').addEventListener('click',bulkReviewSuppressions);$('discoveryWorkspace').addEventListener('click',e=>{const action=e.target.closest?.('[data-run-action]');if(action){const run=discoveryRuns.find(item=>String(item.id)===String(selectedDiscoveryRunId));discoveryAction(action.dataset.runAction,run);}});