User-agent: * # applies to all robots Disallow: /docs/ # disallow indexing of docs directory contents Disallow: /pdf/ # disallow indexing of pdf directory contents Disallow: /newsletter/ # disallow indexing of pdf directory contents