# MYCDIC public crawling policy
# Public institutional content is discoverable. Administrative, private,
# application-processing and machine-data endpoints are intentionally excluded.
User-agent: *
Allow: /
Disallow: /institutional-admin/
Disallow: /staff/
Disallow: /interview/
Disallow: /onboarding/
Disallow: /membership/payment/
Disallow: /membership/account/
Disallow: /membership-requirements/
Disallow: /news/default/cms/
Disallow: /.private/
Disallow: /data/
Disallow: /join-us/gateway-config.php
Disallow: /join-us/gateway-bootstrap.php
Disallow: /join-us/submit-application.php
Disallow: /join-us/transfer-file.php
Disallow: /join-us/verify-email.php

User-agent: OAI-SearchBot
Allow: /
Disallow: /institutional-admin/
Disallow: /staff/
Disallow: /interview/
Disallow: /onboarding/
Disallow: /membership/payment/
Disallow: /membership/account/
Disallow: /membership-requirements/
Disallow: /news/default/cms/
Disallow: /.private/
Disallow: /data/
Disallow: /join-us/gateway-config.php
Disallow: /join-us/gateway-bootstrap.php
Disallow: /join-us/submit-application.php
Disallow: /join-us/transfer-file.php
Disallow: /join-us/verify-email.php

Sitemap: https://www.mycdic.org/sitemap.xml
