add docx download to ETSI files

app.py

@@ -347,6 +347,29 @@ def find_document_batch(request: BatchDocRequest):
         search_time=time.time()-start_time
     )
 
+@app.post("/find/docx", tags=["Document Retrieval"], summary="Download an ETSI specification as DOCX",
+          responses={
+              200: {"description": "DOCX file streamed directly"},
+              400: {"description": "DOCX download only supported for ETSI specifications"},
+              404: {"description": "Specification not found or DOCX not available"},
+          })
+def find_document_docx(request: DocRequest):
+    document = request.doc_id
+    if not valid_etsi_spec_format.match(document):
+        raise HTTPException(status_code=400, detail="DOCX download is only supported for ETSI specifications (e.g. '102 221')")
+
+    result = etsi_spec_finder.search_document_docx(document, request.version)
+
+    if not result.endswith(".docx"):
+        raise HTTPException(status_code=404, detail=result)
+
+    return FileResponse(
+        result,
+        filename=os.path.basename(result),
+        media_type="application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+    )
+
+
 @app.post('/search', response_model=KeywordResponse, tags=["Content Search"], summary="Search specifications by keywords", responses={
              200: {
                  "description": "Search completed successfully"

classes.py

@@ -148,4 +148,120 @@ class ETSISpecFinder:
                 if f.endswith(".pdf"):
                     return url2 + release + "/" + f
 
-        return f"Specification {doc_id} not found"
+        return f"Specification {doc_id} not found"
+
+    def _get_wki_id(self, doc_id: str, version: str = None) -> str:
+        """Return the ETSI portal wki_id for a spec version, or None if not found."""
+        if version:
+            version_str = version
+        else:
+            # Derive version from the FTP PDF URL
+            pdf_url = self.search_document(doc_id)
+            if "not found" in pdf_url.lower():
+                return None
+            # URL path: .../18.04.00_60/ts_...p.pdf  →  folder is parts[-2]
+            parts = pdf_url.rstrip("/").split("/")
+            version_folder = parts[-2]          # e.g. "18.04.00_60"
+            v_parts = version_folder.split("_")[0].split(".")  # ["18", "04", "00"]
+            try:
+                version_str = f"{int(v_parts[0])}.{int(v_parts[1])}.{int(v_parts[2])}"
+            except (ValueError, IndexError):
+                return None
+
+        for spec_type in ["TS", "TR"]:
+            params = {
+                "option": "com_standardssearch",
+                "view": "data",
+                "format": "json",
+                "page": "1",
+                "search": f"ETSI {spec_type} {doc_id} v{version_str}",
+                "etsiNumber": "1",
+                "published": "1",
+            }
+            try:
+                resp = requests.get("https://www.etsi.org/", params=params,
+                                    headers=self.headers, verify=False, timeout=15)
+                data = resp.json()
+                if data and isinstance(data, list):
+                    return str(data[0]["wki_id"])
+            except Exception as e:
+                print(f"Error getting wki_id for {doc_id}: {e}")
+        return None
+
+    def _authenticate_eol(self, wki_id: str) -> requests.Session:
+        """Create a requests.Session authenticated to the ETSI EOL portal."""
+        session = requests.Session()
+        session.headers.update({"User-Agent": self.headers["User-Agent"]})
+
+        login_redir_url = (
+            f"https://portal.etsi.org/LoginRedirection.aspx"
+            f"?ReturnUrl=%2fwebapp%2fprotect%2fNTaccount.asp%3fWki_Id%3d{wki_id}"
+            f"&Wki_Id={wki_id}"
+        )
+        # Seed DNN session cookies
+        session.get(login_redir_url, verify=False, timeout=15)
+
+        # Authenticate via EOL JSON login
+        session.post(
+            "https://portal.etsi.org/ETSIPages/LoginEOL.ashx",
+            data=json.dumps({"username": os.environ.get("EOL_USER"),
+                             "password": os.environ.get("EOL_PASSWORD")}),
+            headers={"Content-Type": "application/json; charset=UTF-8",
+                     "Referer": login_redir_url},
+            verify=False,
+            allow_redirects=False,
+            timeout=15,
+        )
+        return session
+
+    def search_document_docx(self, doc_id: str, version: str = None) -> str:
+        """Download an ETSI spec as DOCX and return the local file path."""
+        wki_id = self._get_wki_id(doc_id, version)
+        if not wki_id:
+            return f"Specification {doc_id} not found"
+
+        session = self._authenticate_eol(wki_id)
+
+        # NTaccount.asp → parse profile_id from meta-refresh
+        r = session.get(
+            f"https://portal.etsi.org/webapp/protect/NTaccount.asp?Wki_Id={wki_id}",
+            verify=False, timeout=15,
+        )
+        meta_match = re.search(r'URL=([^"\'\s>]+)', r.text)
+        if not meta_match:
+            return f"Specification {doc_id}: authentication failed"
+
+        meta_url = meta_match.group(1)
+        if not meta_url.startswith("http"):
+            meta_url = f"https://portal.etsi.org/webapp/protect/{meta_url}"
+
+        # CheckIdentifier → 302 to copy_file
+        r2 = session.get(meta_url, allow_redirects=False, verify=False, timeout=15)
+        if r2.status_code != 302:
+            return f"Specification {doc_id}: download chain failed"
+
+        # copy_file (may have a second redirect)
+        copy_url = "https://portal.etsi.org" + r2.headers["Location"]
+        r3 = session.get(copy_url, allow_redirects=False, verify=False, timeout=15)
+
+        if r3.status_code == 302:
+            final_url = "https://portal.etsi.org/webapp/ewp/" + r3.headers["Location"]
+            r4 = session.get(final_url, verify=False, timeout=15)
+        else:
+            r4 = r3
+
+        # Extract DOCX link
+        docx_urls = re.findall(r'href=["\']([^"\']*\.docx)["\']', r4.text, re.IGNORECASE)
+        if not docx_urls:
+            return f"Specification {doc_id}: DOCX not available"
+
+        docx_url = docx_urls[0]
+
+        # Download
+        dl = session.get(docx_url, headers={"Referer": r4.url}, verify=False, timeout=60)
+        filename = docx_url.split("/")[-1]
+        tmp_path = f"/tmp/{filename}"
+        with open(tmp_path, "wb") as f:
+            f.write(dl.content)
+
+        return tmp_path

static/script.js

@@ -49,10 +49,14 @@ const GPP_SPEC_RE  = /^\d{2}\.\d{3}/;
 
 function toggleVersionField() {
     const docId = document.getElementById('doc-id').value.trim();
-    const group  = document.getElementById('single-version-group');
-    const isSpec = ETSI_SPEC_RE.test(docId) || GPP_SPEC_RE.test(docId);
-    group.style.display = isSpec ? 'block' : 'none';
-    if (!isSpec) document.getElementById('doc-version').value = '';
+    const versionGroup = document.getElementById('single-version-group');
+    const formatGroup  = document.getElementById('single-format-group');
+    const isSpec     = ETSI_SPEC_RE.test(docId) || GPP_SPEC_RE.test(docId);
+    const isEtsiSpec = ETSI_SPEC_RE.test(docId);
+    versionGroup.style.display = isSpec     ? 'block' : 'none';
+    formatGroup.style.display  = isEtsiSpec ? 'block' : 'none';
+    if (!isSpec)     document.getElementById('doc-version').value = '';
+    if (!isEtsiSpec) document.getElementById('doc-format').value  = 'pdf';
 }
 
 // Keyboard shortcuts management
@@ -73,15 +77,23 @@ function setupKeyboardHandlers() {
 // Search functions
 async function searchSingle() {
     const docId = document.getElementById('doc-id').value.trim();
-    
+
     if (!docId) {
         showError('Please enter a document ID');
         return;
     }
-    
+
+    const format = document.getElementById('doc-format').value || 'pdf';
+    const isDocx = ETSI_SPEC_RE.test(docId) && format === 'docx';
+
+    if (isDocx) {
+        downloadDocx(docId);
+        return;
+    }
+
     showLoading();
     updateHeaderStats('Searching...');
-    
+
     try {
         const version = document.getElementById('doc-version').value.trim() || null;
         const body = { doc_id: docId };
@@ -94,9 +106,9 @@ async function searchSingle() {
             },
             body: JSON.stringify(body)
         });
-        
+
         const data = await response.json();
-        
+
         if (response.ok) {
             displaySingleResult(data);
             updateHeaderStats(`Found in ${data.search_time.toFixed(3)}s`);
@@ -113,6 +125,49 @@ async function searchSingle() {
     }
 }
 
+async function downloadDocx(docId) {
+    showLoading();
+    updateHeaderStats('Downloading DOCX...');
+
+    try {
+        const version = document.getElementById('doc-version').value.trim() || null;
+        const body = { doc_id: docId };
+        if (version) body.version = version;
+
+        const response = await fetch('/find/docx', {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify(body)
+        });
+
+        if (response.ok) {
+            const blob = await response.blob();
+            const disposition = response.headers.get('Content-Disposition') || '';
+            const fnMatch = disposition.match(/filename="?([^";\n]+)"?/);
+            const filename = fnMatch ? fnMatch[1] : `${docId.replace(/ /g, '_')}.docx`;
+            const url = URL.createObjectURL(blob);
+            const a = document.createElement('a');
+            a.href = url;
+            a.download = filename;
+            document.body.appendChild(a);
+            a.click();
+            document.body.removeChild(a);
+            URL.revokeObjectURL(url);
+            updateHeaderStats('DOCX downloaded');
+        } else {
+            const data = await response.json();
+            showError(data.detail);
+            updateHeaderStats('Error');
+        }
+    } catch (error) {
+        showError('Error connecting to server');
+        updateHeaderStats('Error');
+        console.error('Error:', error);
+    } finally {
+        hideLoading();
+    }
+}
+
 async function searchBatch() {
     const batchText = document.getElementById('batch-ids').value.trim();
     

templates/index.html

@@ -43,6 +43,13 @@
                         <label for="doc-version">Version <span class="label-hint">(leave empty for latest)</span></label>
                         <input type="text" id="doc-version" placeholder="e.g., 17.6.0">
                     </div>
+                    <div class="form-group version-group" id="single-format-group" style="display:none;">
+                        <label for="doc-format">Format <span class="label-hint">(ETSI specs only)</span></label>
+                        <select id="doc-format">
+                            <option value="pdf">PDF (public)</option>
+                            <option value="docx">DOCX (requires ETSI credentials)</option>
+                        </select>
+                    </div>
                 </div>
 
                 <!-- Batch Search -->