##### Remark. "*": Wild card. "$": End (End with previous letters) ## SECTION 1: ============== PART 1: Google =============== User-agent: Googlebot Disallow: /Sitemap/* ## All use /SitemapBing/ <-- Goodle sitemap == /Sitemap/; Bing sitemap == /SitemapBing/ ##Disallow: /PDF-Excerpt/ShowPDFExcerpt.aspx* Disallow: /m/* ##Disallow: /PDF-English-Translation/* ## Managed by TagX ##Disallow: /PDF/mEnglish.aspx/* ## Split for Mobile Disallow: /PDF.aspx?PDF=* ## Prevent duplicate ##Disallow: /PDF/Sample.aspx/* ## Managed by TagX ##Disallow: /PDF/BOOK.aspx/* ## Managed by TagX Disallow: /CN/* ## Moved to /CHN/ Disallow: /CHN/* ## 2018-04 Disallow: /X10/* ## 2018-05 Disallow: /X10 ## 2018-05 Disallow: /PDF-English-Translation-302/* ## Backup folder Disallow: /PDF-English-Translation-GTran_HTML/* ## Backup folder Disallow: /CUSTOMER-DOWNLOAD/* ## Temp folder Disallow: /TEST/* ## Test folder ##Disallow: /Img/* ## Open to prevent img error Disallow: /Cart/* Disallow: /CN/Cart/* Disallow: /FlexPaper/* Disallow: /FancyBox/* Disallow: /Cookie/* Disallow: /Public* Disallow: /AD* Disallow: /PDF-English-Translat/* ## Obsolete folder Disallow: /PDF-Translated-English/* ## Obsolete folder Disallow: /List-PDF/GBT2.asp* Disallow: /China/* ## Purchase folder ## robots=noinex Disallow: /CHINA/* ## Purchase folder ## robots=noinex Disallow: /TEST/* Disallow: /*.vb$ ##Disallow: *.vb ## This might block ALL Disallow: /Default.aspx?PDF-English-ID=*.html$ ## To remedy the sitemap mistake Disallow: /*.aspx?StdID=* ##Disallow: /*.aspx?RelatedID=* ## SECTION 2: ============== PART 2: Bing =============== User-agent: Bingbot Disallow: /SitemapGonly/* Disallow: /Sitemap/* ## All use /SitemapBing/ <-- Goodle sitemap == /Sitemap/; Bing sitemap == /SitemapBing/ Disallow: /CHN/* ## Google /CHN/ Disallow: /CUSTOMER-DOWNLOAD/* ## Temp folder ## Disallow: /PDF-English-Translation/* Disallow: /PDF-English-Translation-302/* ## Backup folder Disallow: /PDF-English-Translation-GTran_HTML/* ## Backup folder Disallow: /TEST/* ## Test folder ##Disallow: /Img/* Disallow: /Cart/* Disallow: /CN/Cart/* Disallow: /FlexPaper/* Disallow: /FancyBox/* Disallow: /Cookie/* Disallow: /Public_ Disallow: /AD* Disallow: /Public_PDF/* Disallow: /English-Translation-PDF/* ## Obsolete folder Disallow: /PDF-English-Translat/* ## Obsolete folder Disallow: /PDF-Translated-English/* ## Obsolete folder Disallow: /List-PDF/GBT2.asp* Disallow: /China/* ## Purchase folder ## robots=noinex Disallow: /CHINA/* ## Purchase folder ## robots=noinex Disallow: /TEST/* Disallow: /*.vb$ ##Disallow: *.vb ## This might block ALL Disallow: /Default.aspx?PDF-English-ID=*.html$ ## To remedy the sitemap mistake Disallow: /*.aspx?StdID=* Disallow: /*.aspx?RelatedID=* ## SECTION 3: ============ PART 3: Baidu ================== User-agent: Baiduspider Disallow: /SitemapGonly/* ##Disallow: /Sitemap/* ## All use /SitemapBing/ <-- Goodle sitemap == /Sitemap/; Bing sitemap == /SitemapBing/ Disallow: /Default.aspx?* ## Ban Baidu to crawle English pages Disallow: /List-PDF/* ## Ban Baidu to crawle English pages Disallow: /CN/* Disallow: /CHN/* ## Use /China/ Disallow: /CUSTOMER-DOWNLOAD/* ## Temp folder ## Disallow: /PDF-English-Translation/* Disallow: /PDF-English-Translation-302/* ## Backup folder Disallow: /PDF-English-Translation-GTran_HTML/* ## Backup folder Disallow: /TEST/* ## Test folder ##Disallow: /Img/* Disallow: /Cart/* Disallow: /CN/Cart/* Disallow: /FlexPaper/* Disallow: /FancyBox/* Disallow: /Cookie/* Disallow: /Public_ Disallow: /AD* Disallow: /Public_PDF/* Disallow: /English-Translation-PDF/* ## Obsolete folder Disallow: /PDF-English-Translat/* ## Obsolete folder Disallow: /PDF-Translated-English/* ## Obsolete folder Disallow: /List-PDF/GBT2.asp* Disallow: /China/* ## Purchase folder ## robots=noinex Disallow: /TEST/* Disallow: /*.vb$ ##Disallow: *.vb ## This might block ALL Disallow: /Default.aspx?PDF-English-ID=*.html$ ## To remedy the sitemap mistake Disallow: /*.aspx?StdID=* Disallow: /*.aspx?RelatedID=* Disallow: /PDF-Excerpt Disallow: /PDF-Excerpt/ ## SECTION 4: ALL spiders ============== PART 4: Other spiders: ALL =============== User-agent: * ## Important Line !! Disallow: /SitemapGonly/* Disallow: /CHN/* ## Use /China/ ## Disallow: /PDF-English-Translation/* Disallow: /PDF-English-Translation-302/* ## Backup folder Disallow: /PDF-English-Translation-GTran_HTML/* ## Backup folder Disallow: /CUSTOMER-DOWNLOAD/* ## Temp folder Disallow: /TEST/* ## Test folder ##Disallow: /Img/* Disallow: /Cart/* Disallow: /CN/Cart/* Disallow: /FlexPaper/* Disallow: /FancyBox/* Disallow: /Cookie/* Disallow: /Public_ Disallow: /AD* Disallow: /Public_PDF/* Disallow: /English-Translation-PDF/* ## Obsolete folder Disallow: /PDF-English-Translat/* ## Obsolete folder Disallow: /PDF-Translated-English/* ## Obsolete folder Disallow: /List-PDF/GBT2.asp* Disallow: /China/* ## Purchase folder ## robots=noindex Disallow: /TEST/* Disallow: /*.vb$ ##Disallow: *.vb ## This might block ALL Disallow: /Default.aspx?PDF-English-ID=*.html$ ## To remedy the sitemap mistake Disallow: /*.aspx?StdID=* Disallow: /*.aspx?RelatedID=* Disallow: /PDF-Excerpt Disallow: /PDF-Excerpt/ ## SECTION 2: China Spiders ============ PART 5 ================== ## User-agent: sogou spider ## Disallow: /Default.asp* ## Disallow: /PDF-English-Translat/ ## Disallow: /List-PDF/ ## User-agent: sohu agent ## Disallow: /Default.asp* ## Disallow: /PDF-English-Translat/ ## Disallow: /List-PDF/ ## User-agent: iaskspider ## Disallow: /Default.asp* ## Disallow: /PDF-English-Translat/ ## Disallow: /List-PDF/ ## User-agent: YodaoBot ## Disallow: /Default.asp* ## Disallow: /PDF-English-Translat/ ## Disallow: /List-PDF/ ## User-agent: Sogou Orion spider/3.0(+http://www.sogou.com/docs/help/webmasters.htm#07) ## Disallow: / ## User-agent: Sogou web spider/3.0(+http://www.sogou.com/docs/help/webmasters.htm#07) ## Disallow: / ## User-agent: Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07) ## Disallow: /