Merge pull request #76 from Maria-Aidarus/main

API Microservice
slightly modified simplifyContent() to preserve more tags
2023-12-05 09:21:52 +05:30 · 2023-12-04 23:57:46 +03:00 · 2023-12-04 23:48:24 +03:00 · 2023-12-04 23:48:11 +03:00 · 2023-12-04 23:43:59 +03:00 · 2023-12-04 23:38:18 +03:00
--- a/netlify.toml
+++ b/netlify.toml
@ -0,0 +1,2 @@
+[build]
+  functions = "netlify/functions/"
--- a/netlify/functions/form.html
+++ b/netlify/functions/form.html
@ -0,0 +1,128 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <title>Invoke Netlify Function</title>
+    <style>
+        body {
+            font-family: Arial, sans-serif;
+            background-color: #f5f5f5;
+            text-align: center;
+            margin: 0;
+            padding: 0;
+            display: flex;
+            flex-direction: column;
+            justify-content: center;
+            align-items: center;
+            height: 100vh;
+        }
+
+        h1 {
+            color: #000;
+        }
+
+        form {
+            background-color: #fff;
+            padding: 20px;
+            border-radius: 5px;
+            box-shadow: 0px 0px 10px rgba(0, 0, 0, 0.2);
+            max-width: 400px;
+            width: 100%;
+            text-align: left; /* Left-align form elements */
+        }
+
+        label {
+            display: block;
+            margin-bottom: 8px;
+            color: #555;
+            font-weight: bold;
+        }
+
+        input[type="text"] {
+            width: 100%;
+            padding: 12px;
+            margin-bottom: 20px;
+            border: 1px solid #ccc;
+            border-radius: 4px;
+            box-sizing: border-box;
+        }
+
+        .button-container {
+            display: flex;
+            justify-content: center;
+        }
+
+        input[type="submit"] {
+            background-color: #3fbfff;
+            color: #fff;
+            border: none;
+            padding: 12px 24px;
+            cursor: pointer;
+            border-radius: 4px;
+            font-weight: bold;
+            margin: 0 auto; /* Center the button horizontally */
+        }
+
+        input[type="submit"]:hover {
+            background-color: #1177e4;
+        }
+
+        #result {
+            margin-top: 20px;
+            padding: 10px;
+            background-color: transparent;
+            border: none;
+            max-width: 400px;
+            width: 100%;
+            white-space: nowrap; /* Prevent wrapping to the next line */
+            /* overflow: hidden; Hide the overflow */ 
+            text-align: center; /* Center-align text in result div */
+            margin-left: auto; /* Move to the center horizontally */
+            margin-right: auto; /* Move to the center horizontally */
+        }
+    </style>
+</head>
+<body>
+    <h1>Invoke Netlify Function</h1>
+    <form id="invokeForm" action="https://main--radiant-entremet-3fce83.netlify.app/.netlify/functions/handleMetadata" method="POST" onsubmit="invokeFunction(event)">
+        <label for="url">URL:</label>
+        <input type="text" id="url" name="url" value="https://example.com">
+        
+        <label for="apiKey">API Key:</label>
+        <input type="text" id="apiKey" name="apiKey" value="your-api-key">
+
+        <div class="button-container">
+            <input type="submit" value="Invoke Function">
+        </div>
+    </form>
+
+    <div id="result"></div>
+
+    <script>
+        async function invokeFunction(event) {
+            event.preventDefault(); // Prevent the default form submission behavior
+
+            const url = document.getElementById("url").value;
+            const apiKey = document.getElementById("apiKey").value;
+
+            const formData = { url, apiKey };
+
+            try {
+                const response = await fetch("https://main--radiant-entremet-3fce83.netlify.app/.netlify/functions/handleMetadata", {
+                    method: "POST",
+                    headers: {
+                        "Content-Type": "application/json"
+                    },
+                    body: JSON.stringify(formData)
+                });
+
+                const data = await response.json();
+
+                // Display the result in the "result" div
+                document.getElementById("result").innerHTML = JSON.stringify(data, null, 2);
+            } catch (error) {
+                console.error("Error:", error);
+            }
+        }
+    </script>
+</body>
+</html>
--- a/netlify/functions/handleMetadata.js
+++ b/netlify/functions/handleMetadata.js
@ -0,0 +1,170 @@
+const fetch = require('node-fetch'); // Import for webscraping in fetchContentFromURL()
+import { OpenAIApi, Configuration } from 'openai';
+// const { Configuration, OpenAIApi } = require('openai');
+
+// Function to fetch content from URL using a web scraping service
+async function fetchContentFromURL(url) {
+    try {
+        const response = await fetch(url);
+        if (!response.ok) {
+            throw new Error(`HTTP error! status: ${response.status}`);
+        }
+        return await response.text();
+    } catch (error) {
+        console.error(`Could not fetch content from URL: ${error}`);
+        throw error;
+    }
+}
+
+function simplifyContent(content) {
+    // Preserve the title tag and its content
+    let title = content.match(/<title.*?>(.*?)<\/title>/i);
+    title = title ? title[1] : '';
+
+    // Extract the body content, if present
+    let bodyContent = '';
+    const bodyMatch = content.match(/<body.*?>([\s\S]*)<\/body>/i);
+    if (bodyMatch) {
+        bodyContent = bodyMatch[1];
+    } else {
+        // If no body tag, assume entire content is body
+        bodyContent = content;
+    }
+
+    // Remove script and style elements and their content
+    bodyContent = bodyContent.replace(/<script.*?>.*?<\/script>/gms, '');
+    bodyContent = bodyContent.replace(/<style.*?>.*?<\/style>/gms, '');
+
+    // Remove all remaining HTML tags, except for title, body, h1-h6, p, and a
+    bodyContent = bodyContent.replace(/<(?!\/?(title|body|h[1-6]|p|a)( [^>]*)?>)([^>]+)>/g, '');
+
+    // Manually replace common HTML entities
+    bodyContent = bodyContent
+        .replace(/&amp;/g, '&')
+        .replace(/&lt;/g, '<')
+        .replace(/&gt;/g, '>')
+        .replace(/&quot;/g, '"')
+        .replace(/&#39;/g, "'");
+
+    // Remove inline CSS and JavaScript event handlers
+    bodyContent = bodyContent.replace(/style\s*=\s*'.*?'/gi, '');
+    bodyContent = bodyContent.replace(/on\w+\s*=\s*".*?"/gi, '');
+
+    // Normalize whitespace without removing sentence punctuation
+    bodyContent = bodyContent.replace(/\s+/g, ' ').trim();
+
+    // Condense multiple line breaks into a single one
+    bodyContent = bodyContent.replace(/(\r\n|\r|\n){2,}/g, '\n');
+
+    // Reconstruct content with title and body
+    const simplifiedContent = `<title>${title}</title><body>${bodyContent}</body>`;
+    return simplifiedContent;
+}
+
+// Placeholder function to perform GPT analysis for media type and topics using Mistral-7b via OpenRouter
+async function performGPTAnalysis(simplifiedContent, apiKey) {
+    // Implement logic to send content to Mistral-7b via OpenRouter for GPT analysis
+    // Send content and receive GPT analysis response
+
+    // this is the code that we tried to use for the GPT Analysis 
+    // try {
+    //     const configuration = new Configuration({
+    //         apiKey: apiKey,  // Use the provided API key
+    //         baseURL: "https://openrouter.ai/api/v1" // Your custom API endpoint
+    //     });
+    
+    //     const openai = new OpenAIApi(configuration);
+
+    //     // Using the specified prompt
+    //      const prompt = `
+    //      Analyze the following text for content categorization:
+    //      Text: "${simplifiedContent}"
+    //      Please provide:
+    //      1. The most likely media type (e.g., article, book, audio, video, chat, research_paper, wiki, etc.)
+    //      2. Key topics covered in the text (list up to 5 main topics).
+    //      `;
+    //     const completion = await openai.createCompletion({
+    //         model: "mistralai/mistral-7b-instruct",
+    //         prompt: prompt,
+    //         max_tokens: 150 // Adjust as needed
+    //     });
+
+    //     //return completion.data.choices[0].text.trim();
+    //     return inferredMediaType;
+    // } catch (error) {
+    //     console.error('Error with OpenAI completion:', error);
+    //     throw error;
+    // }
+    // however, it gives the error below: 
+    // { "error": "Something went wrong", "details": "Configuration is not a constructor" }
+
+    // Placeholder code
+    const inferredMediaType = ["article"];
+    const extractedTopics = ["topic1", "topic2"];
+    return { inferredMediaType, extractedTopics };
+}
+
+// Placeholder function to map inferred values to predefined formats and topics
+function mapInferredValues(mediaType, topics) {
+    // Implement logic to map inferred media type and topics to predefined formats and topics
+    // Match inferred values with predefined taxonomy
+    // Placeholder code
+    const predefinedMediaType = "Article";
+    const predefinedTopics = ["Topic 1", "Topic 2"];
+    return { predefinedMediaType, predefinedTopics };
+}
+
+// Placeholder function to format the response
+function formatResponse(predefinedMediaType, predefinedTopics) {
+    // Implement logic to format the extracted metadata into the desired response structure
+    // Construct the response object
+    // Placeholder code
+    const response = {
+        format: predefinedMediaType,
+        topics: predefinedTopics,
+    };
+    return response;
+}
+
+export async function handler(event) {
+    try {
+        // Extract URL and API Key from the request body
+        const { url, apiKey } = JSON.parse(event.body);
+
+        // Validate if URL and API Key are present
+        if (!url || !apiKey) {
+            return {
+                statusCode: 400,
+                body: JSON.stringify({ error: 'URL and API Key are required' }),
+            };
+        }
+
+        // Step 1: Fetch content from the URL using a web scraping service
+        const fetchedContent = await fetchContentFromURL(url);
+
+        // Step 2: Simplify the fetched content for GPT analysis
+        const simplifiedContent = simplifyContent(fetchedContent);
+
+        // Step 3: Perform GPT analysis for media type and topics
+        const { inferredMediaType, extractedTopics } = await performGPTAnalysis(simplifiedContent, apiKey);
+
+        // Step 4: Map inferred values to predefined formats and topics
+        const { predefinedMediaType, predefinedTopics } = mapInferredValues(inferredMediaType, extractedTopics);
+
+        // Step 5: Format the response
+        const formattedResponse = formatResponse(predefinedMediaType, predefinedTopics);
+
+        // Return the formatted response
+        return {
+            statusCode: 200,
+            // returning the output of the simplifyContent function, to test the function
+            body: JSON.stringify(simplifiedContent),
+        };
+    } catch (error) {
+        console.error('Error occurred:', error.message);
+        return {
+            statusCode: 500,
+            body: JSON.stringify({ error: 'Something went wrong', details: error.message }),
+        };
+    }
+}
--- a/netlify/functions/package-lock.json
+++ b/netlify/functions/package-lock.json
@ -0,0 +1,299 @@
+{
+  "name": "handle-metadata-function",
+  "version": "1.0.0",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "": {
+      "name": "handle-metadata-function",
+      "version": "1.0.0",
+      "dependencies": {
+        "he": "^1.2.0",
+        "node-fetch": "^2.7.0",
+        "openai": "4.20.0"
+      }
+    },
+    "node_modules/@types/node": {
+      "version": "18.18.13",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-18.18.13.tgz",
+      "integrity": "sha512-vXYZGRrSCreZmq1rEjMRLXJhiy8MrIeVasx+PCVlP414N7CJLHnMf+juVvjdprHyH+XRy3zKZLHeNueOpJCn0g==",
+      "dependencies": {
+        "undici-types": "~5.26.4"
+      }
+    },
+    "node_modules/@types/node-fetch": {
+      "version": "2.6.9",
+      "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.9.tgz",
+      "integrity": "sha512-bQVlnMLFJ2d35DkPNjEPmd9ueO/rh5EiaZt2bhqiSarPjZIuIV6bPQVqcrEyvNo+AfTrRGVazle1tl597w3gfA==",
+      "dependencies": {
+        "@types/node": "*",
+        "form-data": "^4.0.0"
+      }
+    },
+    "node_modules/abort-controller": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz",
+      "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==",
+      "dependencies": {
+        "event-target-shim": "^5.0.0"
+      },
+      "engines": {
+        "node": ">=6.5"
+      }
+    },
+    "node_modules/agentkeepalive": {
+      "version": "4.5.0",
+      "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.5.0.tgz",
+      "integrity": "sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==",
+      "dependencies": {
+        "humanize-ms": "^1.2.1"
+      },
+      "engines": {
+        "node": ">= 8.0.0"
+      }
+    },
+    "node_modules/asynckit": {
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
+      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
+    },
+    "node_modules/base-64": {
+      "version": "0.1.0",
+      "resolved": "https://registry.npmjs.org/base-64/-/base-64-0.1.0.tgz",
+      "integrity": "sha512-Y5gU45svrR5tI2Vt/X9GPd3L0HNIKzGu202EjxrXMpuc2V2CiKgemAbUUsqYmZJvPtCXoUKjNZwBJzsNScUbXA=="
+    },
+    "node_modules/charenc": {
+      "version": "0.0.2",
+      "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz",
+      "integrity": "sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==",
+      "engines": {
+        "node": "*"
+      }
+    },
+    "node_modules/combined-stream": {
+      "version": "1.0.8",
+      "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
+      "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
+      "dependencies": {
+        "delayed-stream": "~1.0.0"
+      },
+      "engines": {
+        "node": ">= 0.8"
+      }
+    },
+    "node_modules/crypt": {
+      "version": "0.0.2",
+      "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz",
+      "integrity": "sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==",
+      "engines": {
+        "node": "*"
+      }
+    },
+    "node_modules/delayed-stream": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
+      "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
+      "engines": {
+        "node": ">=0.4.0"
+      }
+    },
+    "node_modules/digest-fetch": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/digest-fetch/-/digest-fetch-1.3.0.tgz",
+      "integrity": "sha512-CGJuv6iKNM7QyZlM2T3sPAdZWd/p9zQiRNS9G+9COUCwzWFTs0Xp8NF5iePx7wtvhDykReiRRrSeNb4oMmB8lA==",
+      "dependencies": {
+        "base-64": "^0.1.0",
+        "md5": "^2.3.0"
+      }
+    },
+    "node_modules/event-target-shim": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz",
+      "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==",
+      "engines": {
+        "node": ">=6"
+      }
+    },
+    "node_modules/form-data": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
+      "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
+      "dependencies": {
+        "asynckit": "^0.4.0",
+        "combined-stream": "^1.0.8",
+        "mime-types": "^2.1.12"
+      },
+      "engines": {
+        "node": ">= 6"
+      }
+    },
+    "node_modules/form-data-encoder": {
+      "version": "1.7.2",
+      "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz",
+      "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A=="
+    },
+    "node_modules/formdata-node": {
+      "version": "4.4.1",
+      "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz",
+      "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==",
+      "dependencies": {
+        "node-domexception": "1.0.0",
+        "web-streams-polyfill": "4.0.0-beta.3"
+      },
+      "engines": {
+        "node": ">= 12.20"
+      }
+    },
+    "node_modules/formdata-node/node_modules/web-streams-polyfill": {
+      "version": "4.0.0-beta.3",
+      "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz",
+      "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==",
+      "engines": {
+        "node": ">= 14"
+      }
+    },
+    "node_modules/he": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz",
+      "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==",
+      "bin": {
+        "he": "bin/he"
+      }
+    },
+    "node_modules/humanize-ms": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz",
+      "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==",
+      "dependencies": {
+        "ms": "^2.0.0"
+      }
+    },
+    "node_modules/is-buffer": {
+      "version": "1.1.6",
+      "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz",
+      "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w=="
+    },
+    "node_modules/md5": {
+      "version": "2.3.0",
+      "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz",
+      "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==",
+      "dependencies": {
+        "charenc": "0.0.2",
+        "crypt": "0.0.2",
+        "is-buffer": "~1.1.6"
+      }
+    },
+    "node_modules/mime-db": {
+      "version": "1.52.0",
+      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
+      "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
+    "node_modules/mime-types": {
+      "version": "2.1.35",
+      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
+      "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
+      "dependencies": {
+        "mime-db": "1.52.0"
+      },
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
+    "node_modules/ms": {
+      "version": "2.1.3",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
+      "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
+    },
+    "node_modules/node-domexception": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz",
+      "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/jimmywarting"
+        },
+        {
+          "type": "github",
+          "url": "https://paypal.me/jimmywarting"
+        }
+      ],
+      "engines": {
+        "node": ">=10.5.0"
+      }
+    },
+    "node_modules/node-fetch": {
+      "version": "2.7.0",
+      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
+      "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
+      "dependencies": {
+        "whatwg-url": "^5.0.0"
+      },
+      "engines": {
+        "node": "4.x || >=6.0.0"
+      },
+      "peerDependencies": {
+        "encoding": "^0.1.0"
+      },
+      "peerDependenciesMeta": {
+        "encoding": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/openai": {
+      "version": "4.20.0",
+      "resolved": "https://registry.npmjs.org/openai/-/openai-4.20.0.tgz",
+      "integrity": "sha512-VbAYerNZFfIIeESS+OL9vgDkK8Mnri55n+jN0UN/HZeuM0ghGh6nDN6UGRZxslNgyJ7XmY/Ca9DO4YYyvrszGA==",
+      "dependencies": {
+        "@types/node": "^18.11.18",
+        "@types/node-fetch": "^2.6.4",
+        "abort-controller": "^3.0.0",
+        "agentkeepalive": "^4.2.1",
+        "digest-fetch": "^1.3.0",
+        "form-data-encoder": "1.7.2",
+        "formdata-node": "^4.3.2",
+        "node-fetch": "^2.6.7",
+        "web-streams-polyfill": "^3.2.1"
+      },
+      "bin": {
+        "openai": "bin/cli"
+      }
+    },
+    "node_modules/tr46": {
+      "version": "0.0.3",
+      "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
+      "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="
+    },
+    "node_modules/undici-types": {
+      "version": "5.26.5",
+      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
+      "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA=="
+    },
+    "node_modules/web-streams-polyfill": {
+      "version": "3.2.1",
+      "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz",
+      "integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==",
+      "engines": {
+        "node": ">= 8"
+      }
+    },
+    "node_modules/webidl-conversions": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
+      "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="
+    },
+    "node_modules/whatwg-url": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
+      "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==",
+      "dependencies": {
+        "tr46": "~0.0.3",
+        "webidl-conversions": "^3.0.0"
+      }
+    }
+  }
+}
--- a/netlify/functions/package.json
+++ b/netlify/functions/package.json
@ -0,0 +1,14 @@
+{
+  "name": "handle-metadata-function",
+  "version": "1.0.0",
+  "description": "A Netlify function to handle metadata extraction and analysis",
+  "main": "handleMetadata.js",
+  "scripts": {
+    "start": "node handleMetadata.js"
+  },
+  "dependencies": {
+    "he": "^1.2.0",
+    "node-fetch": "^2.7.0",
+    "openai": "^4.20.0"
+  }
+}
--- a/package-lock.json
+++ b/package-lock.json
--- a/package.json
+++ b/package.json
@ -23,7 +23,7 @@
    "svelte-bricks": "^0.2.1",
    "svelte-preprocess": "^5.0.4",
    "tailwindcss": "^3.3.3",
-    "vite": "^4.4.9"
+    "vite": "^4.5.0"
  },
  "overrides": {
    "@rgossiaux/svelte-headlessui": {
@ -35,6 +35,10 @@
  },
  "dependencies": {
    "@tailwindcss/typography": "^0.5.10",
+    "firebase-admin": "^11.11.1",
+    "firebase-functions-test": "^3.1.0",
+    "node-fetch": "^2.7.0",
+    "openai": "^4.20.0",
    "svelvet": "8.1.0"
  }
 }
Autor	SHA1	Wiadomość	Data
Nilesh	c26bb6dbe7	Merge pull request #76 from Maria-Aidarus/main API Microservice	2023-12-05 09:21:52 +05:30
Huda Joad	ded050d616	slightly modified simplifyContent() to preserve more tags	2023-12-04 23:57:46 +03:00
Huda Joad	d9735112c5	preserve body tag	2023-12-04 23:48:24 +03:00
Huda Joad	7de9f17ec3	preserve body tag	2023-12-04 23:48:11 +03:00
Huda Joad	5b3c6578aa	modified simplifyContent()	2023-12-04 23:43:59 +03:00
Huda Joad	9d8827ab52	changed the function a little to keep the tags	2023-12-04 23:38:18 +03:00
Maria	186d6c22dd	Merge pull request #6 from Maria-Aidarus/updatePrompt Updated prompt	2023-12-01 16:09:04 +03:00
aishaalsubaie	c7399df455	updated prompt	2023-12-01 15:05:58 +03:00
Skultrix	55e084f11b	Deleted public index.html	2023-11-30 12:54:28 +03:00
Skultrix	f9d89ec9e8	Deleted firebase.json	2023-11-30 12:53:58 +03:00
Skultrix	03f806eb0a	Deleted firebase file	2023-11-30 12:53:40 +03:00
Skultrix	201cfd6b99	Deleted last firebase workflow	2023-11-30 12:53:12 +03:00
Skultrix	01624b8745	Deleted firebase workflow	2023-11-30 12:53:00 +03:00
Skultrix	3de7afb69b	Deleted package.json	2023-11-30 12:52:16 +03:00
Skultrix	b1d577101f	Deleted index.js	2023-11-30 12:51:53 +03:00
Skultrix	7e3d8242d7	Deleted gitignore	2023-11-30 12:51:10 +03:00
Maria-Aidarus	4af057d61d	updated the dependencies	2023-11-29 18:13:39 +03:00
Maria-Aidarus	8c3506bdcc	fixed the firebase deployment service	2023-11-29 18:06:24 +03:00
Maria-Aidarus	326f4bd897	commented out the try/catch block in the gpt analysis function	2023-11-29 17:57:22 +03:00
Maria-Aidarus	801cabdce9	changed the structure of the imports	2023-11-29 12:22:24 +03:00
Maria-Aidarus	5478b7709f	changed the structure of the imports	2023-11-29 12:18:49 +03:00
Maria-Aidarus	18e430969e	chnaged the formatting	2023-11-28 21:47:22 +03:00
Maria-Aidarus	3a1689058e	updated the package.json file	2023-11-28 21:11:25 +03:00
Maria-Aidarus	edf8f0e41a	updated the package.json file	2023-11-28 21:05:23 +03:00
Maria-Aidarus	92e01d67c9	chnaged the import again	2023-11-28 20:52:26 +03:00
Maria-Aidarus	2993b45a35	added the ai model functionality	2023-11-28 20:44:54 +03:00
Maria-Aidarus	c6857eb355	chnaged the syntax of the import	2023-11-28 20:42:36 +03:00
Maria-Aidarus	586030a79a	removed the import	2023-11-28 20:41:01 +03:00
Maria-Aidarus	4cf2d5e1d1	changed the import	2023-11-28 20:38:34 +03:00
Maria-Aidarus	c00c8f0277	added the try/catch block back	2023-11-28 20:34:21 +03:00
Maria-Aidarus	d83b03cc09	removed the functionality from the gpt analysis function	2023-11-28 20:29:12 +03:00
Maria-Aidarus	e864cdb99a	updated the paramters for the gpt function	2023-11-27 13:02:56 +03:00
Maria-Aidarus	bb868258af	removed the overflow option for the form	2023-11-26 15:14:12 +03:00
Huda Joad	c46f1b0f28	edited comments	2023-11-25 01:01:18 +03:00
Huda Joad	5145b0fe3e	trying another way to simplify	2023-11-25 00:59:25 +03:00
Huda Joad	e6b9cfc6dd	using node-fetch again cause apparently that wasnt the issue	2023-11-25 00:55:28 +03:00
Huda Joad	0056258711	changed to use https again in fetchContentFromURL()	2023-11-25 00:52:48 +03:00
Huda Joad	5cbb0f208c	fixed syntax error	2023-11-25 00:49:06 +03:00
Huda Joad	45e4a696ac	changed import type	2023-11-25 00:47:35 +03:00
Huda Joad	ab4060003b	added he back for testing	2023-11-25 00:44:52 +03:00
Huda Joad	6e6d44fc16	removed he as it is not really needed, just helpful, but gave runtime error	2023-11-25 00:42:25 +03:00
Huda Joad	950a4aee3c	changed import format	2023-11-25 00:38:00 +03:00
Huda Joad	c4b0159d22	new import: he	2023-11-25 00:36:04 +03:00
Huda Joad	d2354929ee	changed back cause nothing different	2023-11-25 00:31:16 +03:00
Huda Joad	ff46e8f8bb	checking if it works without stringify	2023-11-25 00:29:46 +03:00
Huda Joad	a9a0d26746	added in special handling for headings in simplifyContent()	2023-11-25 00:26:54 +03:00
Huda Joad	73443e4ce4	modified the simplifyContent function to retain punctuation to make it more suitable for LLM analysis	2023-11-25 00:21:55 +03:00
Huda Joad	cf1833e684	modified simplifyContent to make it cover more html	2023-11-25 00:15:52 +03:00
Huda Joad	259e45451f	modified simplifyContent()	2023-11-25 00:10:29 +03:00
Huda Joad	9cdf9e6e45	changed openai import	2023-11-25 00:09:19 +03:00
Huda Joad	d8d5b0db71	changed back to previous version	2023-11-25 00:07:18 +03:00
Huda Joad	4a5bb87b0e	modified simplifyContent()	2023-11-25 00:03:43 +03:00
Huda Joad	b866b6e7d0	updated the response to test simplifyContent()	2023-11-24 23:56:28 +03:00
Huda Joad	a603edd790	updated package.json to type module, updated import to match commonjs syntax	2023-11-24 23:53:32 +03:00
Huda Joad	9cd0345098	downgraded node-fetch version	2023-11-24 23:49:12 +03:00
Huda Joad	60c6bc5e75	modified catch error block to give error message	2023-11-24 23:44:21 +03:00
Huda Joad	535ccfaefe	modified fetchContentFromURL() to use node-fetch now that import is working	2023-11-24 23:38:02 +03:00
Huda Joad	2bad128834	checking if fetchedContent() works correctly	2023-11-24 23:35:00 +03:00
Maria-Aidarus	5dc0645826	trying node-fetch now	2023-11-24 21:01:43 +03:00
Maria-Aidarus	e34158bcb5	updated the function, uses node-fetch now	2023-11-24 20:59:37 +03:00
Maria-Aidarus	96a2bfb87c	trying node-fetch now	2023-11-24 20:52:38 +03:00
Maria-Aidarus	7e38977885	updated the package.json file	2023-11-24 20:40:49 +03:00
Maria-Aidarus	cd5fa89f41	moved the form.html file	2023-11-24 20:34:24 +03:00
Maria-Aidarus	6e097f8351	moved the form.html file	2023-11-24 20:33:26 +03:00
Maria-Aidarus	e68e858fca	inshallah this works	2023-11-24 20:10:33 +03:00
Maria-Aidarus	4026cb5889	inshallah this works	2023-11-24 20:10:14 +03:00
Maria-Aidarus	1f0f249b72	added openai to netlify.toml	2023-11-24 20:01:03 +03:00
Maria-Aidarus	7ff1cde46d	inshallah	2023-11-24 19:54:56 +03:00
Maria-Aidarus	7da53fb064	removed the install script	2023-11-24 19:51:37 +03:00
Maria-Aidarus	b73bc63922	removed the install script	2023-11-24 19:43:40 +03:00
Maria-Aidarus	f0b3468fe2	removed the install script	2023-11-24 19:43:05 +03:00
Maria-Aidarus	021922cbe2	removed the install script	2023-11-24 19:39:40 +03:00
Maria-Aidarus	9825e2056c	added an install script	2023-11-24 19:36:44 +03:00
Maria-Aidarus	369e633114	added an install script	2023-11-24 19:36:13 +03:00
Maria-Aidarus	ac2d27efef	updated the deployment settings	2023-11-24 19:31:50 +03:00
Maria-Aidarus	32c4b15eea	updated the deployment settings	2023-11-24 19:30:35 +03:00
Maria-Aidarus	9ab0543fcc	updated the deployment settings	2023-11-24 19:29:58 +03:00
Maria-Aidarus	9e84ec5daa	moved everything into the same folder	2023-11-24 19:24:14 +03:00
Maria-Aidarus	4dcfe72834	moved everything into the same folder	2023-11-24 19:22:29 +03:00
Maria-Aidarus	9f0821905d	tried to fix the openai import	2023-11-24 19:16:13 +03:00
Huda Joad	aae2faafe4	changed package.json	2023-11-24 18:58:15 +03:00
Huda Joad	fe364b9bec	changed openai import	2023-11-24 18:57:57 +03:00
Huda Joad	e302e41c75	added new package.json	2023-11-24 18:54:43 +03:00
Huda Joad	28473643f0	import openai	2023-11-24 18:49:26 +03:00
Huda Joad	34da18949d	changed netlify.toml	2023-11-24 18:47:34 +03:00
Huda Joad	f0064a6d7c	removed default	2023-11-24 18:46:38 +03:00
Huda Joad	2c4bf059c5	changed netlify.toml	2023-11-24 18:44:45 +03:00
Huda Joad	4cb262b0ec	changed netlify.toml	2023-11-24 18:43:08 +03:00
Huda Joad	c29406a1c6	added node-fetch	2023-11-24 18:41:59 +03:00
Huda Joad	feb3843e27	was returning the wrong thing	2023-11-24 18:40:13 +03:00
Huda Joad	17f6bbc08e	not summarizing	2023-11-24 18:39:31 +03:00
Huda Joad	e591ed32cb	removed imports	2023-11-24 18:37:45 +03:00
Huda Joad	ef73d65548	axios	2023-11-24 18:35:06 +03:00
Huda Joad	90e0f02440	request node-fetch	2023-11-24 18:33:46 +03:00
Huda Joad	ccf5ced811	changed netlify.toml	2023-11-24 18:32:14 +03:00
Huda Joad	ccb2ae11f6	changed netlify.toml	2023-11-24 18:31:16 +03:00
Huda Joad	ed6740782e	changed netlify.toml	2023-11-24 18:29:31 +03:00
Huda Joad	72c94b6de5	added npm install to netlify.toml and added node-fetch back in	2023-11-24 18:25:26 +03:00
Huda Joad	bfc24807a8	downgraded node-fetch	2023-11-24 18:22:24 +03:00
Huda Joad	39549d150a	removing node-fetch	2023-11-24 18:17:58 +03:00
Huda Joad	81175f245b	adding node-fetch	2023-11-24 18:16:33 +03:00
Huda Joad	fe353adc01	removing node-fetch	2023-11-24 18:13:43 +03:00
Huda Joad	2a8de7a16e	adding node-fetch back in	2023-11-24 18:08:28 +03:00
Huda Joad	4b9ea1b124	removing node-fetch import	2023-11-24 18:07:19 +03:00
Huda Joad	b367e0c2a6	adding node-fetch back in	2023-11-24 18:04:27 +03:00
Huda Joad	ced286735a	removing node-fetch import	2023-11-24 18:03:02 +03:00
Huda Joad	29a2116293	trying node-fetch with .default	2023-11-24 18:00:43 +03:00
Huda Joad	71516c962c	added in axios again	2023-11-24 17:54:43 +03:00
Huda Joad	7ac1609187	testing without axios	2023-11-24 17:52:56 +03:00
Huda Joad	b7e4ba611c	trying with axios instead of node-fetch	2023-11-24 17:50:59 +03:00
Huda Joad	238963ad60	changed node-fetch version type, trying to make it work	2023-11-24 17:43:27 +03:00
Huda Joad	ec711e2df4	trying to add node-fetch again	2023-11-24 17:40:20 +03:00
Huda Joad	70e817e2c0	improving simplifyContent()	2023-11-24 17:34:13 +03:00
Huda Joad	3dfa222849	added stuff to get rid of formatting from html	2023-11-24 17:26:49 +03:00
Huda Joad	3132902590	added a link to remove links	2023-11-24 17:19:52 +03:00
Huda Joad	c0ea5e0997	testing simplifiedContent function	2023-11-24 17:14:17 +03:00
Huda Joad	a4da9cbf3f	changed the format back	2023-11-24 17:10:48 +03:00
Huda Joad	734d989a12	added simplyContent functionality (not tested yet); changed the return to test the format	2023-11-24 17:05:17 +03:00
Huda Joad	d9418bdb14	trying to use https import for webscraping instead of node-fetch	2023-11-23 19:48:59 +03:00
Huda Joad	5888f7b0d7	changed node-fetch version with uninstall and reinstalling, added node-fetch import back in	2023-11-23 19:46:06 +03:00
Huda Joad	2195b4ed01	changed version of node-fetch back; import https instead of node-fetch	2023-11-23 19:43:07 +03:00
Huda Joad	e90b3288e9	changed node-fetch version in package.json	2023-11-23 19:40:27 +03:00
Huda Joad	01017326a9	changed format of import	2023-11-23 19:35:51 +03:00
Huda Joad	eb2fcf4854	added back in the node-fetch import	2023-11-23 19:32:58 +03:00
Huda Joad	2cdcfef121	return the fetchedContent instead of formattedResponse	2023-11-23 19:30:37 +03:00
Maria-Aidarus	8cc6e36210	reverted the handleMetadata file	2023-11-23 19:26:25 +03:00
Maria-Aidarus	510bad2a47	cried	2023-11-23 19:23:35 +03:00
Huda Joad	fbf55af8b0	updated the return for testing	2023-11-23 19:21:14 +03:00
Maria-Aidarus	1905bd7d39	removed all imports	2023-11-23 19:17:56 +03:00
Huda Joad	104ca57098	testing if the imports are broken	2023-11-23 19:16:07 +03:00
Maria-Aidarus	ae8186ad36	removed openai from the handleMetadata file	2023-11-23 19:12:28 +03:00
Huda Joad	6ab782c3fe	added openai to package.json file	2023-11-23 19:08:42 +03:00
Huda Joad	5952faafff	Merge branch 'main' of https://github.com/Maria-Aidarus/learndb	2023-11-23 19:07:53 +03:00
Maria-Aidarus	002bba1e3a	added openai to the package.json file	2023-11-23 19:02:07 +03:00
Huda Joad	c3bd6f88de	added openai to package.json	2023-11-23 18:53:29 +03:00
Maria-Aidarus	a713eaedaf	added node-fetch to the package.json file	2023-11-23 18:49:47 +03:00
Huda Joad	338e155146	Merge pull request #4 from Maria-Aidarus/web-scraping Web scraping function test	2023-11-23 18:43:38 +03:00
Huda Joad	c0cb0a978f	accidentally removed a comma	2023-11-23 18:42:06 +03:00
Huda Joad	f68436d5c2	added functionality for fetchContentFromURL(); added some notes for performGPTAnalysis(); modified some other lines to test fetchContentFromURL(); didn't test if anything works yetgit add .!	2023-11-23 17:43:12 +03:00
Maria-Aidarus	4f8dddfd97	updated the css on the form.html page	2023-11-22 23:44:21 +03:00
Maria-Aidarus	0ea53c276d	updated the css on the form.html page	2023-11-22 23:07:52 +03:00
Maria-Aidarus	3e2020af2c	updated the css on the form.html page	2023-11-22 23:06:00 +03:00
Maria-Aidarus	b27909a84b	updated the css on the form.html page	2023-11-22 23:01:43 +03:00
Maria-Aidarus	7e5cb70dcd	updated the css on the form.html page	2023-11-22 22:58:39 +03:00
Maria-Aidarus	4e1809280c	updated the css on the form.html page	2023-11-22 22:57:09 +03:00
Maria-Aidarus	312b2e686a	updated the css on the form.html page	2023-11-22 22:51:56 +03:00
Maria-Aidarus	4a8c0e5a2c	updated the css on the form.html page	2023-11-22 22:46:16 +03:00
Maria	88e27db2d9	Merge pull request #3 from Maria-Aidarus/netlify Updated file location	2023-11-22 22:40:23 +03:00
Maria-Aidarus	e48296378e	moved the form.html to another directory	2023-11-22 22:36:22 +03:00
Maria	9681f2c54d	Merge pull request #2 from Maria-Aidarus/netlify Created an HTML file for the Netlify function.	2023-11-22 22:25:03 +03:00
Maria-Aidarus	7c97630d01	created an html form that can take in the url and api key	2023-11-22 22:11:49 +03:00
Maria	126cad3307	Merge pull request #1 from Maria-Aidarus/netlify Netlify	2023-11-22 15:45:39 +03:00
Maria-Aidarus	ecc7a01f14	Updated the handleMetadata.js	2023-11-22 15:33:41 +03:00
Maria-Aidarus	15c09238a4	Created the netlify.toml file and the file to handle the metadata	2023-11-22 14:41:01 +03:00