# Robots file for site no_NO for domain www

# Apply rules to all user agents
User-agent: *

# START of standard rules

# My History links exclusion
Disallow: /ctl/do/savetolist
Disallow: /ctl/do/deletefromlist

# Special folders exclusion
Disallow: /modules/
Disallow: /customs/
Disallow: /content/
Disallow: /b2b/
Disallow: /merchant/
Disallow: /Bs/
Disallow: /bargains/
Disallow: /jscripts/
Disallow: /shopbot/
Disallow: /css/
Disallow: /users/
Disallow: /servlets/
Disallow: /servlet/
Disallow: /admin/
Disallow: /data/
Disallow: /sitesearch/
Disallow: /pdbCultural/
Disallow: /mobile
Disallow: /pcc/
Disallow: /ni/
Disallow: /nf/

# Leads pages exclusion
Disallow: /go.jsp
Disallow: /waitingPage.jsp
Disallow: /midishopbot?
Disallow: /resultsmidi
Disallow: /ctl/go
Disallow: /ctl/do/resultsmidi
Disallow: /ctl/do/midi
Disallow: /b/a/midishopbot

# Reviews and ratings exclusion
Disallow: /ctl/do/editProductReview
Disallow: /ctl/do/sendProductReview
Disallow: /ctl/do/previewProductReview
Disallow: /ctl/do/voteForReview
Disallow: /ctl/do/displayReportAbuseProductReview
Disallow: /ctl/do/displayReportAbuseMerchantReview
Disallow: /ctl/do/sendMerchantReview
Disallow: /ctl/do/previewMerchantReview
Disallow: /ctl/do/editMerchantReview
Disallow: /ctl/do/quickRateProduct
Disallow: /maxiMerchantReviews?
Disallow: /mr-*?
Disallow: /*/mrl?

# Former JSP pages exclusion
Disallow: /search.jsp
Disallow: /next.jsp
Disallow: /category.jsp
Disallow: /row_gateway.jsp

# Obsolete pages exclusion
Disallow: /o_
Disallow: /ctl/do/search
Disallow: /ctl/do/compare
Disallow: /ctl/do/compareNext
Disallow: /ctl/do/sort
Disallow: /ctl/do/compareProducts
Disallow: /ctl/do/selectProducts
Disallow: /ctl/do/content
Disallow: /ctl/do/bargainHomepage
Disallow: /ctl/do/browse
Disallow: /ctl/exportMerchant

# Other pages exclusion
Disallow: /results~
Disallow: /ctl/do/error404
Disallow: /ctl/do/error403
Disallow: /ctl/do/errorSession
Disallow: /ctl/do/productNotFound
Disallow: /ctl/do/asyncCall
Disallow: /resultsmini~
Disallow: /suggestAsYouType.html
Disallow: /ctl/TdG2

# Malformed urls exclusion
Disallow: /b/a/drilldown
Disallow: /b/a/results~
Disallow: /b/a/resultsmaxi~
Disallow: /b/a/resultsmidi~

# Travel forms exclusion
Disallow: /ctl/do/checkFlight
Disallow: /ctl/do/tripPlannerCheckForm
Disallow: /ctl/do/tripPlannerCheck
Disallow: /ctl/do/checkMultiCityForm
Disallow: /ctl/do/azkelkooform
Disallow: /ctl/do/azkelkoo
Disallow: /ctl/do/moduleDisplay

# END of standard rules

# START of NO rules

# Cultural live shopbot pages exclusion
Disallow: /sbs/no/music/

# Travel A-Z pages exclusion (avoid duplicate content)
Disallow: /170201/

# NO root domains rules
Disallow: /m-3267801-bindersno.html

# END of NO rules

# START of NO rules

# NO shopping domains rules
Disallow: /ss-lege- hans-jacob-gravdal,-sandefjord.html
Disallow: /ss-Oslo-porno-anal.html
Disallow: /ss-Anal-fuck.html
Disallow: /ss-Film-vhs-porno-norge.html
Disallow: /ss-anal-sex.html
Disallow: /ss-brazil-young-girls-sex.html
Disallow: /ss-Sex-foto.html
Disallow: /ss-fitte.html
Disallow: /ss-Erotik-pics.html
Disallow: /ss-escortservice-mallorca.html
Disallow: /ss-Paris-hilton-sex-movie.html
Disallow: /ss-hot-celeberti-sex.html
Disallow: /ss-Disney-sex.html
Disallow: /ss-Sex-stakuk-samleie.html
Disallow: /ss-Gratis-sex-bilder.html
Disallow: /ss-riga-sex.html

# END of NO rules

# Parameters pages exclusion
Disallow: /ss-*?
Disallow: /ssc-*?
Disallow: /ssm-*?
Disallow: /sscm-*?
Disallow: /p-*?
Disallow: /ajax/
Disallow: /slp/
Disallow: /offer-*
Disallow: /ads/
Disallow: *?t=
Disallow: *?*&t=

# Ensure the /SLP pages are crawlable by the AdX/AdSense crawler, to ensure proper targeting of those pages 
User-agent: Mediapartners-Google

Allow: /slp/


# Majestic SEO
User-Agent: MJ12bot

Disallow: 

# Ensure all pages are crawlable by Bing
User-Agent: AdIdxBot
Disallow:

# Disallow SemrushBot
User-agent: SemrushBot
Disallow: /
User-agent: SemrushBot-SA
Disallow: /