InsightConnect Marketplace

urlscan.io

Back to Marketplace

urlscan.io

v2.1.2

Scan your website

Tags: url, analysis, intel


Actions
  • Get Scan Results
  • Search
  • Submit URL for Scan

urlscan.io

About

Urlscan is a free URL scanning service.

This plugin utilizes the Urlscan API.

Actions

This action is used to search urlscan.io.

Input

Name Type Default Required Description Enum
q string None True The query term (ElasticSearch simple query string) e.g domain:urlscan.io, default is * None
sort string _score True Sorting, specificied via $sort_field:$sort_order None
size integer 100 True Number of results returned None
offset integer 0 True Offset of first result (for paginating) None

Output

Name Type Required Description
total integer False None
results []results False UrlScan.io Results

Example output:

[
   {
      "_id":"9b135a8b-d6ba-4d39-a93c-de0cd4378dcc",
      "page":{
         "asn":"AS24940",
         "asnname":"HETZNER-AS, DE",
         "city":"",
         "country":"DE",
         "domain":"urlscan.io",
         "ip":"148.251.45.170",
         "ptr":"urlscan.io",
         "server":"nginx",
         "url":"https://urlscan.io/"
      },
      "result":"https://urlscan.io/api/v1/result/9b135a8b-d6ba-4d39-a93c-de0cd4378dcc",
      "stats":{
         "consoleMsgs":0,
         "dataLength":835633,
         "encodedDataLength":296287,
         "requests":30,
         "uniqIPs":6
      },
      "task":{
         "method":"api",
         "source":"api",
         "time":"2019-01-03T16:00:28.529Z",
         "url":"http://urlscan.io",
         "visibility":"public"
      },
      "uniq_countries":2
   }
]

Submit URL for Scan

This action is used to submit a url to generate a scan report that can be retrieved later.

Input

Name Type Default Required Description Enum
url string None True The URL to scan None
public boolean False True Set to false for a private scan None

Output

Name Type Required Description
scan_id string True UUID of the scan to query later

Example output:

{
  "scan_id": "557a7923-c597-4a84-982c-665ece8fa6ed"}
}

Get Scan Results

This action is used to get the results of a scan.

Input

Name Type Default Required Description Enum
scan_id string None True UUID of the scan to retrieve None

Output

Name Type Required Description
scan_results scan_results True Results of the scan report

Example output:

{
    "scan_results": {
        "requests": [
            {
                "request": {
                    "requestId": "47A4C275E4F9888784CC56FB194AC950",
                    "loaderId": "47A4C275E4F9888784CC56FB194AC950",
                    "documentURL": "https://www.google.com/?gws_rd=ssl",
                    "request": {
                        "url": "https://www.google.com/?gws_rd=ssl",
                        "method": "GET",
                        "headers": {
                            "Upgrade-Insecure-Requests": "1",
                            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36"
                        },
                        "mixedContentType": "none",
                        "initialPriority": "VeryHigh",
                        "referrerPolicy": "no-referrer-when-downgrade"
                    },
                    "timestamp": 30579913.395083,
                    "wallTime": 1546278138.193554,
                    "initiator": {
                        "type": "other"
                    },
                    "redirectResponse": {
                        "url": "http://www.google.com/",
                        "status": 302,
                        "statusText": "Found",
                        "headers": {
                            "Location": "https://www.google.com/?gws_rd=ssl",
                            "Cache-Control": "private",
                            "Content-Type": "text/html; charset=UTF-8",
                            "P3P": "CP=\"This is not a P3P policy! See g.co/p3phelp for more info.\"",
                            "Date": "Mon, 31 Dec 2018 17:42:18 GMT",
                            "Server": "gws",
                            "Content-Length": "231",
                            "X-XSS-Protection": "1; mode=block",
                            "X-Frame-Options": "SAMEORIGIN",
                            "Set-Cookie": "1P_JAR=2018-12-31-17; expires=Wed, 30-Jan-2019 17:42:18 GMT; path=/; domain=.google.com\nNID=152=LSC4Vcy981xZ6F9BrZAaF97wP1t8VKPLhPkBHnU5wG7ZfQDpurZphSUPpw4T3ErINvKmLpFIxrCfyzhtXHBDhlrJ5G412FYdCaEiSet37hsN5YmBbfUhBj5UjmzdSLwOLBY_T1tYis2rd-hTr12etNJ78s5N5NU7_MeNg408s0Y; expires=Tue, 02-Jul-2019 17:42:18 GMT; path=/; domain=.google.com; HttpOnly\nCONSENT=WP.2752ea; expires=Fri, 01-Jan-2038 00:00:00 GMT; path=/; domain=.google.com"
                        },
                        "mimeType": "text/html",
                        "requestHeaders": {
                            "Host": "www.google.com",
                            "Connection": "keep-alive",
                            "Pragma": "no-cache",
                            "Cache-Control": "no-cache",
                            "Upgrade-Insecure-Requests": "1",
                            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36",
                            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
                            "Accept-Encoding": "gzip, deflate"
                        },
                        "remoteIPAddress": "[2a00:1450:4001:820::2004]",
                        "remotePort": 80,
                        "encodedDataLength": 803,
                        "timing": {
                            "requestTime": 30579913.345846,
                            "proxyStart": -1,
                            "proxyEnd": -1,
                            "dnsStart": -1,
                            "dnsEnd": -1,
                            "connectStart": -1,
                            "connectEnd": -1,
                            "sslStart": -1,
                            "sslEnd": -1,
                            "workerStart": -1,
                            "workerReady": -1,
                            "sendStart": 0.519,
                            "sendEnd": 0.547,
                            "pushStart": 0,
                            "pushEnd": 0,
                            "receiveHeadersEnd": 48.809
                        },
                        "protocol": "http/1.1",
                        "securityState": "neutral"
                    },
                    "type": "Document",
                    "frameId": "1DEBC07916E13CAB920D1FF70099D072",
                    "hasUserGesture": false
                }
            }
        ],
        "cookies": [
            {
                "name": "NID",
                "value": "152=SXZkTJ2B0OtwUgtuhshFOwo6pZpdK_wT392bPKcfDvT8f6F3rAUFzdUHOZZZHUliugUP9ObePB8GoXkOtP7sPQMEupWyL0y5WrUIrBKlaXP3ZIpwDxhQuz9XqTJGKtz56Z9zzDFX5wcfepb5dKacMCtpQAdSrMPabeC2Idu538Y",
                "domain": ".google.com",
                "path": "/",
                "expires": 1562089338.309606,
                "size": 178,
                "httpOnly": true,
                "secure": false,
                "session": false
            }
        ],
        "console": [],
        "links": [
            {
                "href": "https://store.google.com/?utm_source=hp_header&utm_medium=google_oo&utm_campaign=GS100042",
                "text": "Store"
            }
        ],
        "timing": {
            "beginNavigation": "2018-12-31T17:42:18.143Z",
            "frameStartedLoading": "2018-12-31T17:42:18.312Z",
            "frameNavigated": "2018-12-31T17:42:18.314Z",
            "domContentEventFired": "2018-12-31T17:42:18.425Z",
            "loadEventFired": "2018-12-31T17:42:18.795Z",
            "frameStoppedLoading": "2018-12-31T17:42:18.795Z"
        },
        "globals": [
            {
                "prop": "onselectstart",
                "type": "object"
            },
            {
                "prop": "onselectionchange",
                "type": "object"
            }
        ],
        "screenshotURL": "google.com"
    }
}

Triggers

This plugin does not contain any triggers.

Connection

The connection configuration accepts the following parameters:

Name Type Default Required Description Enum
api_key credential_secret_key None False urlscan API key. Not required for the search action None

Troubleshooting

This plugin does not contain any troubleshooting information.

Workflows

Examples:

  • URL enrichment

Versions

  • 0.1.0 - Initial plugin
  • 0.1.1 - SSL bug fix in SDK
  • 1.0.0 - Update to v2 Python plugin architecture | Support web server mode
  • 2.0.0 - Fixed issue where output of Get Scan Results did not match API output | Update connection input to secret key instead of token | Updates to help
  • 2.1.0 - Added ScreenshotURL to get scan results output
  • 2.1.1 - Add error messaging to Get Scan Results action to provide assistance for unavailable scan results | Update to Python 3.7 Slim SDK (plugin size reduction)
  • 2.1.2 - Set User-Agent string to Rapid7 InsightConnect | Update to use the komand/python-3-37-slim-plugin:3 Docker image to reduce plugin size | Run plugin as least privileged user | Improve error handling and logging | Fix issue in Submit URL for Scan action where improper POST body was sent

References

plugin_spec_version: v2
extension: plugin
product: ["insightconnect"]
name: urlscan
title: urlscan.io
vendor: rapid7
support: community
status: []
description: Scan your website
version: 2.1.2
resources:
  source_url: https://github.com/rapid7/insightconnect-plugins/
  license_url: https://github.com/rapid7/insightconnect-plugins/blob/master/LICENSE
tags:
- url
- analysis
- intel
types:
  page:
    asn:
      description: Autonomous System Number
      type: string
      required: false
    asnname:
      description: Autonomous System Name
      type: string
      required: false
    city:
      type: string
      required: false
    country:
      type: string
      required: false
    domain:
      type: string
      required: false
    ip:
      description: IP address
      type: string
      required: false
    ptr:
      type: string
      required: false
    server:
      description: Server Software
      type: string
      required: false
    url:
      type: string
      required: false
  stats:
    consoleMsgs:
      description: Console Messages
      type: integer
      required: false
    dataLength:
      description: Data Length
      type: integer
      required: false
    encodedDataLength:
      description: Encoded Data Length
      type: integer
      required: false
    requests:
      type: integer
      required: false
    uniqIPs:
      description: Unique IPs
      type: integer
      required: false
  task:
    method:
      type: string
      required: false
    options:
      type: object
      required: false
    source:
      type: string
      required: false
    time:
      type: date
      required: false
    url:
      type: string
      required: false
    visibility:
      type: string
      required: false
  results:
    _id:
      type: string
      required: false
    page:
      type: page
      required: false
    result:
      type: string
      required: false
    stats:
      type: stats
      required: false
    task:
      type: task
      required: false
    uniq_countries:
      description: Unique Countries
      type: integer
      required: false
  scan_results:
    requests:
      type: '[]object'
      required: false
    cookies:
      type: '[]object'
      required: false
    console:
      type: '[]object'
      required: false
    links:
      type: '[]object'
      required: false
    timing:
      type: object
      required: false
    globals:
      type: '[]object'
      required: false
    screenshotURL:
      type: string
      required: false
connection:
  api_key:
    title: API Key for Saved Scans
    description: urlscan API key. Not required for the search action
    type: credential_secret_key
    required: false
actions:
  search:
    title: Search
    description: Search urlscan.io
    input:
      q:
        title: Query
        description: The query term (ElasticSearch simple query string) e.g domain:urlscan.io,
          default is *
        type: string
        required: true
      size:
        title: Size
        description: Number of results returned
        type: integer
        default: 100
        required: true
      offset:
        title: Offset
        description: Offset of first result (for paginating)
        type: integer
        default: 0
        required: true
      sort:
        title: Sort
        description: Sorting, specificied via $sort_field:$sort_order
        type: string
        default: _score
        required: true
    output:
      total:
        title: Total
        type: integer
        description: Total number of results returned
        required: false
      results:
        title: Results
        description: UrlScan.io Results
        type: '[]results'
        required: false
  submit_url_for_scan:
    title: Submit URL for Scan
    description: Submit a URL to generate a scan report that can be retrieved later
    input:
      url:
        title: URL
        type: string
        description: The URL to scan
        required: true
      public:
        type: boolean
        title: Public
        description: Set to false for a private scan
        required: true
        default: false
    output:
      scan_id:
        title: Scan ID
        type: string
        description: UUID of the scan to query later
        required: true
  get_scan_results:
    title: Get Scan Results
    description: Get the results of a scan
    input:
      scan_id:
        title: Scan ID
        type: string
        description: UUID of the scan to retrieve
        required: true
    output:
      scan_results:
        title: Scan Results
        description: Results of the scan report
        type: scan_results
        required: true
Other plugins
BMC Remedy ITSM
Rapid7   |   v1.7.0
Plugin
Get
String Operations
Rapid7   |   v1.2.0
Plugin
Get
Microsoft Teams
Rapid7   |   v1.2.2
Plugin
Get
HTML
Rapid7   |   v1.2.0
Plugin
Get
Active Directory LDAP
Rapid7   |   v3.2.6
Plugin
Get