# www.mortgageauditsonline.com
#
# Served as a static file from public/ rather than app/robots.js: the app runs
# with `trailingSlash: true`, and a metadata route is one more thing that can
# pick up a 308 to /robots.txt/ — which is not the URL crawlers request.

User-agent: *
Allow: /

# Admin panel and its API (mortgage-site on :4041). Both require a session and
# answer 401 without one; there is nothing here for a crawler.
Disallow: /admin
Disallow: /api/

# Search results — thin, infinite, and duplicated from the archives.
Disallow: /*?s=

# Deleted spam posts. These 404 correctly and should stay 404; blocking the
# crawl only stops the wasted requests, it does not remove them from the index
# (a disallowed URL is never fetched, so a 404 is never seen). Left crawlable
# on purpose.

Sitemap: https://www.mortgageauditsonline.com/sitemap_index.xml