Scrape YouTube Keyword Suggestions Using Node.js Express Puppeteer

npm init -y

npm init -y

npm i express

npm i express

npm i puppeteer

npm i puppeteer

npm i ejs

npm i ejs

npm i google-search-results-nodejs

npm i google-search-results-nodejs

npm i puppeteer-extra

npm i puppeteer-extra

npm i puppeteer-extra-plugin-stealth

npm i puppeteer-extra-plugin-stealth

const puppeteer = require("puppeteer-extra");
const StealthPlugin = require("puppeteer-extra-plugin-stealth");
const ObjectsToCsv = require('objects-to-csv');
const express = require('express')
puppeteer.use(StealthPlugin());
const app = express()
app.use(express.urlencoded({ extended: false }))
app.set('view engine','ejs')
app.get('/', (req, res) => {
    res.render('index',{suggestions:[]})
})

app.post('/getsuggestions', (req, res) => {
    let search = req.body.search.split('\n')
    getYoutubeAutocomplete(search)
    .then((data) => {
        console.log(data)
        res.render('index',{suggestions:data.autocompleteResults})
    })
})
async function getYoutubeAutocomplete(search) {
    const queries = search;
    const URL = "https://www.youtube.com";
    const browser = await puppeteer.launch({
        headless: false,
        args: ["--no-sandbox", "--disable-setuid-sandbox"],
    });

    const page = await browser.newPage();

    await page.setDefaultNavigationTimeout(60000);
    await page.waitForTimeout(4000)

    await page.goto(URL);
    await page.waitForSelector("#contents");

    const autocompleteResults = [];
    for (query of queries) {
        await page.click("#search-input");
        await page.keyboard.type(query);
        await page.waitForTimeout(5000);
        const results = {
            query,
            autocompleteResults: await page.evaluate(() => {
                return Array.from(document.querySelectorAll(".sbdd_a li"))
                    .map((el) => el.querySelector(".sbqs_c")?.textContent.trim())
                    .filter((el) => el);
            }),
        };
        autocompleteResults.push(results);
        await page.click("#search-clear-button");
        await page.waitForTimeout(2000);
    }

    await browser.close();

    const csv = new ObjectsToCsv(autocompleteResults);
    await csv.toDisk('./test.csv');


    return autocompleteResults;
}

app.listen(5000,() => {
    console.log("App is listening on port 5000")
})

const puppeteer = require("puppeteer-extra");

const StealthPlugin = require("puppeteer-extra-plugin-stealth");

const ObjectsToCsv = require('objects-to-csv');

const express = require('express')

puppeteer.use(StealthPlugin());

const app = express()

app.use(express.urlencoded({ extended: false }))

app.set('view engine','ejs')

app.get('/', (req, res) => {

    res.render('index',{suggestions:[]})

})

​

app.post('/getsuggestions', (req, res) => {

    let search = req.body.search.split('\n')

    getYoutubeAutocomplete(search)

    .then((data) => {

        console.log(data)

        res.render('index',{suggestions:data.autocompleteResults})

    })

})

async function getYoutubeAutocomplete(search) {

    const queries = search;

    const URL = "https://www.youtube.com";

    const browser = await puppeteer.launch({

        headless: false,

        args: ["--no-sandbox", "--disable-setuid-sandbox"],

    });

​

    const page = await browser.newPage();

​

    await page.setDefaultNavigationTimeout(60000);

    await page.waitForTimeout(4000)

​

    await page.goto(URL);

    await page.waitForSelector("#contents");

​

    const autocompleteResults = [];

    for (query of queries) {

        await page.click("#search-input");

        await page.keyboard.type(query);

        await page.waitForTimeout(5000);

        const results = {

            query,

            autocompleteResults: await page.evaluate(() => {

                return Array.from(document.querySelectorAll(".sbdd_a li"))

                    .map((el) => el.querySelector(".sbqs_c")?.textContent.trim())

                    .filter((el) => el);

            }),

        };

        autocompleteResults.push(results);

        await page.click("#search-clear-button");

        await page.waitForTimeout(2000);

    }

​

    await browser.close();

​

    const csv = new ObjectsToCsv(autocompleteResults);

    await csv.toDisk('./test.csv');

​

​

    return autocompleteResults;

}

​

app.listen(5000,() => {

    console.log("App is listening on port 5000")

})

<!DOCTYPE html>
<html lang="en">

<head>
    <meta charset="UTF-8">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>Get Youtube Suggestions</title>
</head>

<body>
    <form action="/getsuggestions" method="post">
        <textarea cols="15" rows="15" name="search" required></textarea>
        <button>Get Suggestions</button>
    </form>
    <%if(suggestions){%>

        <table>
            <thead>
                <tr>
                    <th>Suggestions</th>
                </tr>
            </thead>
            <tbody>
                <%suggestions.forEach(item=> {%>
                    <tr>
                        <td>
                            <%=item%>
                        </td>
                    </tr>
                    <%})%>
            </tbody>
        </table>

        <%}%>
</body>

</html>

<!DOCTYPE html>

<html lang="en">

​

<head>

    <meta charset="UTF-8">

    <meta http-equiv="X-UA-Compatible" content="IE=edge">

    <meta name="viewport" content="width=device-width, initial-scale=1.0">

    <title>Get Youtube Suggestions</title>

</head>

​

<body>

    <form action="/getsuggestions" method="post">

        <textarea cols="15" rows="15" name="search" required></textarea>

        <button>Get Suggestions</button>

    </form>

    <%if(suggestions){%>

​

        <table>

            <thead>

                <tr>

                    <th>Suggestions</th>

                </tr>

            </thead>

            <tbody>

                <%suggestions.forEach(item=> {%>

                    <tr>

                        <td>

                            <%=item%>

                        </td>

                    </tr>

                    <%})%>

            </tbody>

        </table>

​

        <%}%>

</body>

​

</html>