User-agent: *
Disallow: /shorting
Disallow: /auth
Disallow: /backend
Disallow: /comps
Disallow: /crawler
Disallow: /maintenance
Disallow: /test
Disallow: /error