Workflow: Schedule Removeduplicates Create

Workflow Details

Download Workflow
{
    "meta": {
        "instanceId": "408f9fb9940c3cb18ffdef0e0150fe342d6e655c3a9fac21f0f644e8bedabcd9",
        "templateCredsSetupCompleted": true
    },
    "nodes": [
        {
            "id": "1083afcb-1257-45a3-b331-d93fb8769548",
            "name": "Schedule Trigger",
            "type": "n8n-nodes-base.scheduleTrigger",
            "position": [
                -840,
                0
            ],
            "parameters": {
                "rule": {
                    "interval": [
                        []
                    ]
                }
            },
            "typeVersion": 1.1999999999999999555910790149937383830547332763671875
        },
        {
            "id": "c3ec0759-a3d1-4866-978a-bfe4f49ee81d",
            "name": "Get Hash of Contents",
            "type": "n8n-nodes-base.crypto",
            "position": [
                380,
                0
            ],
            "parameters": {
                "value": "={{ $json.data }}",
                "dataPropertyName": "hash"
            },
            "typeVersion": 1
        },
        {
            "id": "4ad92ad3-6f99-43a5-9a61-374adb3b28f7",
            "name": "Notify of Change",
            "type": "n8n-nodes-base.gmail",
            "position": [
                1220,
                0
            ],
            "webhookId": "cc4cbee1-ad57-48fb-810a-b21880357ab0",
            "parameters": {
                "sendTo": "jim@example.com",
                "message": "=site: {{ $('Variables').first().json.url }}\ndate: {{ $now.toISO() }}\nhash: {{ $json.hash }}\ncontents: {{ $json.gdrive }}",
                "options": [],
                "subject": "=Change detected for {{ $('Variables').first().json.url }}",
                "emailType": "text"
            },
            "credentials": {
                "gmailOAuth2": {
                    "id": "Sf5Gfl9NiFTNXFWb",
                    "name": "Gmail account"
                }
            },
            "typeVersion": 2.100000000000000088817841970012523233890533447265625
        },
        {
            "id": "9385242a-86b1-4624-ada9-949851b68054",
            "name": "Markdown",
            "type": "n8n-nodes-base.markdown",
            "position": [
                200,
                0
            ],
            "parameters": {
                "html": "={{ $json.content.trim() }}",
                "options": []
            },
            "typeVersion": 1
        },
        {
            "id": "663b6ba2-cbf9-44ea-a062-f88ccb0640c1",
            "name": "Variables",
            "type": "n8n-nodes-base.set",
            "position": [
                -460,
                0
            ],
            "parameters": {
                "options": [],
                "assignments": {
                    "assignments": [
                        {
                            "id": "fa706ed3-7425-4f0e-9590-d9478b4e6692",
                            "name": "url",
                            "type": "string",
                            "value": "https:\/\/x.com\/en\/tos"
                        }
                    ]
                }
            },
            "typeVersion": 3.399999999999999911182158029987476766109466552734375
        },
        {
            "id": "2ef69fff-4d7a-439a-b317-ea5fa3ce7869",
            "name": "Only New Hashes",
            "type": "n8n-nodes-base.removeDuplicates",
            "position": [
                560,
                0
            ],
            "parameters": {
                "options": {
                    "historySize": 1
                },
                "operation": "removeItemsSeenInPreviousExecutions",
                "dedupeValue": "={{ $json.hash }}"
            },
            "typeVersion": 2
        },
        {
            "id": "93e13d80-a26f-408f-be4e-b582ab9fd9bd",
            "name": "Log Record",
            "type": "n8n-nodes-base.googleSheets",
            "position": [
                1040,
                0
            ],
            "parameters": {
                "columns": {
                    "value": {
                        "hash": "={{ $('Get Hash of Contents').first().json.hash }}",
                        "gdrive": "=https:\/\/drive.google.com\/file\/d\/{{ $json.id }}\/view?usp=sharing",
                        "website": "={{ $('Variables').first().json.url }}",
                        "date of change": "={{ $now.toISO() }}"
                    },
                    "schema": [
                        {
                            "id": "website",
                            "type": "string",
                            "display": true,
                            "required": false,
                            "displayName": "website",
                            "defaultMatch": false,
                            "canBeUsedToMatch": true
                        },
                        {
                            "id": "hash",
                            "type": "string",
                            "display": true,
                            "required": false,
                            "displayName": "hash",
                            "defaultMatch": false,
                            "canBeUsedToMatch": true
                        },
                        {
                            "id": "date of change",
                            "type": "string",
                            "display": true,
                            "required": false,
                            "displayName": "date of change",
                            "defaultMatch": false,
                            "canBeUsedToMatch": true
                        },
                        {
                            "id": "gdrive",
                            "type": "string",
                            "display": true,
                            "removed": false,
                            "required": false,
                            "displayName": "gdrive",
                            "defaultMatch": false,
                            "canBeUsedToMatch": true
                        }
                    ],
                    "mappingMode": "defineBelow",
                    "matchingColumns": [],
                    "attemptToConvertTypes": false,
                    "convertFieldsToString": false
                },
                "options": [],
                "operation": "append",
                "sheetName": {
                    "__rl": true,
                    "mode": "list",
                    "value": "gid=0",
                    "cachedResultUrl": "https:\/\/docs.google.com\/spreadsheets\/d\/1K13OBMicH-ebhvJRYo4sHiOuHd1KIX2jiFxBLWy2UOk\/edit#gid=0",
                    "cachedResultName": "Sheet1"
                },
                "documentId": {
                    "__rl": true,
                    "mode": "list",
                    "value": "1K13OBMicH-ebhvJRYo4sHiOuHd1KIX2jiFxBLWy2UOk",
                    "cachedResultUrl": "https:\/\/docs.google.com\/spreadsheets\/d\/1K13OBMicH-ebhvJRYo4sHiOuHd1KIX2jiFxBLWy2UOk\/edit?usp=drivesdk",
                    "cachedResultName": "86. Webpage Changes Tracker"
                }
            },
            "credentials": {
                "googleSheetsOAuth2Api": {
                    "id": "XHvC7jIRR8A2TlUl",
                    "name": "Google Sheets account"
                }
            },
            "typeVersion": 4.5
        },
        {
            "id": "dea15963-61de-4371-b716-7683e0e703a9",
            "name": "Sticky Note",
            "type": "n8n-nodes-base.stickyNote",
            "position": [
                -540,
                -160
            ],
            "parameters": {
                "color": 7,
                "width": 620,
                "height": 380,
                "content": "## 1. Fetch a Webpage Contents\n[Read more about the HTTP request node](https:\/\/docs.n8n.io\/integrations\/builtin\/core-nodes\/n8n-nodes-base.httprequest)"
            },
            "typeVersion": 1
        },
        {
            "id": "bcb1c0b1-fc77-4417-a9c0-6299ce8ca33e",
            "name": "Sticky Note1",
            "type": "n8n-nodes-base.stickyNote",
            "position": [
                100,
                -160
            ],
            "parameters": {
                "color": 7,
                "width": 640,
                "height": 400,
                "content": "## 2. Use Hashing to Detect Changes\n[Learn more about the cryptography node](https:\/\/docs.n8n.io\/integrations\/builtin\/core-nodes\/n8n-nodes-base.crypto\/)"
            },
            "typeVersion": 1
        },
        {
            "id": "eda51de6-80a2-42b1-83ba-66c551911c45",
            "name": "Fetch Webpage",
            "type": "n8n-nodes-base.httpRequest",
            "position": [
                -280,
                0
            ],
            "parameters": {
                "url": "={{ $json.url }}",
                "options": [],
                "sendHeaders": true,
                "headerParameters": {
                    "parameters": [
                        {
                            "name": "User-Agent",
                            "value": "Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/134.0.0.0 Safari\/537.36"
                        }
                    ]
                }
            },
            "typeVersion": 4.20000000000000017763568394002504646778106689453125
        },
        {
            "id": "8aa998f1-1558-4b54-a95a-7be9360a819a",
            "name": "Extract Contents",
            "type": "n8n-nodes-base.html",
            "position": [
                -100,
                0
            ],
            "parameters": {
                "options": [],
                "operation": "extractHtmlContent",
                "extractionValues": {
                    "values": [
                        {
                            "key": "content",
                            "cssSelector": ".ct07",
                            "returnValue": "html"
                        }
                    ]
                }
            },
            "typeVersion": 1.1999999999999999555910790149937383830547332763671875
        },
        {
            "id": "ecb60054-563c-4c0b-949c-abdcd3896f6b",
            "name": "Sticky Note2",
            "type": "n8n-nodes-base.stickyNote",
            "position": [
                760,
                -160
            ],
            "parameters": {
                "color": 7,
                "width": 660,
                "height": 400,
                "content": "## 3. Notify when Changes Occur\n[Read more about the Gmail node](https:\/\/docs.n8n.io\/integrations\/builtin\/app-nodes\/n8n-nodes-base.gmail)\n"
            },
            "typeVersion": 1
        },
        {
            "id": "30774606-5e63-4d73-9894-ba14d5b1fc46",
            "name": "Take a Snapshot",
            "type": "n8n-nodes-base.googleDrive",
            "position": [
                860,
                0
            ],
            "parameters": {
                "name": "={{\n$('Variables').item.json.url\n  .extractDomain()\n  .replace('.','_')\n  + $json.hash\n  + '.md'\n}}",
                "content": "={{ $json.data }}",
                "driveId": {
                    "__rl": true,
                    "mode": "list",
                    "value": "My Drive"
                },
                "options": [],
                "folderId": {
                    "__rl": true,
                    "mode": "list",
                    "value": "1BbP5HRnjNkCvcDRFFg1bm0sNYBGZM6AU",
                    "cachedResultUrl": "https:\/\/drive.google.com\/drive\/folders\/1BbP5HRnjNkCvcDRFFg1bm0sNYBGZM6AU",
                    "cachedResultName": "86. Monitor Webpage Changes"
                },
                "operation": "createFromText"
            },
            "credentials": {
                "googleDriveOAuth2Api": {
                    "id": "yOwz41gMQclOadgu",
                    "name": "Google Drive account"
                }
            },
            "typeVersion": 3
        },
        {
            "id": "9ae563ea-49a3-4dcc-9190-8746650018aa",
            "name": "Sticky Note3",
            "type": "n8n-nodes-base.stickyNote",
            "position": [
                -1040,
                -760
            ],
            "parameters": {
                "width": 440,
                "height": 980,
                "content": "## Try it out\n### This n8n template can monitor and detect changes to a webpage's contents and notify you only when a change occurs.\n\nGreat to keep an eye on and track publicly available documents such as  company TOS, government policy or competitor pages.\n\n### How it works\n* A scheduled trigger is used so we can run everyday to automate this process.\n* A website page is then fetched with the HTTP request node and the contents we want to track are extracted using the HTML node.\n* To detect changes, we generate a hash on the contents with the cryptography node and compare it with previously seen hashes using the \"remove duplicates\" node. If the hash was seen before, the workflow stops here.\n* Finally, when new changes are detected a copy of the contents are uploaded to Google Drive and a logged into a Google sheet. A notification email can also be sent if action is required.\n\n### How to use\n* Update the URL you want to track in the node named \"variables\" and ensure the HTML node has updated selectors to get the content you want.\n* Ensure the timezone is set correctly when using the Scheduled Trigger node.\n\n### Need Help?\nJoin the [Discord](https:\/\/discord.com\/invite\/XPKeKXeB7d) or ask in the [Forum](https:\/\/community.n8n.io\/)!"
            },
            "typeVersion": 1
        }
    ],
    "pinData": [],
    "connections": {
        "Markdown": {
            "main": [
                [
                    {
                        "node": "Get Hash of Contents",
                        "type": "main",
                        "index": 0
                    }
                ]
            ]
        },
        "Variables": {
            "main": [
                [
                    {
                        "node": "Fetch Webpage",
                        "type": "main",
                        "index": 0
                    }
                ]
            ]
        },
        "Log Record": {
            "main": [
                [
                    {
                        "node": "Notify of Change",
                        "type": "main",
                        "index": 0
                    }
                ]
            ]
        },
        "Fetch Webpage": {
            "main": [
                [
                    {
                        "node": "Extract Contents",
                        "type": "main",
                        "index": 0
                    }
                ]
            ]
        },
        "Only New Hashes": {
            "main": [
                [
                    {
                        "node": "Take a Snapshot",
                        "type": "main",
                        "index": 0
                    }
                ]
            ]
        },
        "Take a Snapshot": {
            "main": [
                [
                    {
                        "node": "Log Record",
                        "type": "main",
                        "index": 0
                    }
                ]
            ]
        },
        "Extract Contents": {
            "main": [
                [
                    {
                        "node": "Markdown",
                        "type": "main",
                        "index": 0
                    }
                ]
            ]
        },
        "Schedule Trigger": {
            "main": [
                [
                    {
                        "node": "Variables",
                        "type": "main",
                        "index": 0
                    }
                ]
            ]
        },
        "Get Hash of Contents": {
            "main": [
                [
                    {
                        "node": "Only New Hashes",
                        "type": "main",
                        "index": 0
                    }
                ]
            ]
        }
    }
}
Back to Workflows

Related Workflows

Auto-Tag Blog Posts in WordPress with AI
View
Telegram Mondaycom Automate Triggered
View
Exponential Backoff for Google APIs
View
Manual HTTP Automation Webhook
View
Twilio Stickynote Send Triggered
View
Stock Q&A Workflow
View
Facebookleadads Stickynote Automate Triggered
View
Schedule Slack Create Scheduled
View
Manual Readbinaryfile Automate Triggered
View
Chat with Postgresql Database
View