User-agent: * Disallow: /wp-admin/ Disallow: /wp-includes/ Disallow: /cgi-bin/ # Allow specific files in disallowed directories Allow: /wp-admin/admin-ajax.php # Block duplicate content Disallow: /tag/ Disallow: /author/ # Sitemap location Sitemap: https://smartpati.com/wp-sitemap.xml User-agent: facebookexternalhit Disallow: User-agent: Googlebot Disallow: # Disallow everything for all other crawlers User-agent: * Disallow: /