�PNG  IHDR22?�� pHYs  �� OiCCPPhotoshop ICC profilexڝSgTS�=���BK���KoR RB���&*! J�!��Q�EEȠ�����Q, � ��!���������{�kּ������>���� �H3Q5� �B�������.@� $p�d!s�#�~<<+"��x� �M��0���B�\���t�8K�@z�B�@F���&S�`�cb�P-`'������{[�!�� e�Dh;��V�EX0fK�9�-0IWfH���� � 0Q��){`�##x��F�W<�+��*x��<�$9E�[-qWW.(�I+6aa�@.�y�2�4���������x����6��_-��"bb���ϫp@�t~��,/��;�m��%�h^ �u��f�@����W�p�~<�5�j>{�-�]c�K'Xt����o��(�h���w��?�G�%�fI�q^D$.Tʳ?�D��*�A��,���� �`6�B$��BB d�r`)��B(�Ͱ*`/�@4�Qh��p.�U�=p�a��(�� A�a!ڈb�X#����!�H�$ ɈQ"K�5H1R�T UH�=r9�\F��;�2����G1���Q=� �C��7�F� �dt1�����r�=�6��Ыhڏ>C�0��3�l0.��B�8, �c˱"� ���V����cϱw�E� 6wB aAHXLXN�H� $4� 7 �Q�'"��K�&���b21�XH,#��/{�C�7$�C2'��I��T��F�nR#�,��4H#���dk�9�, +ȅ����3��!�[ �b@q��S�(R�jJ��4�e�2AU��Rݨ�T5�ZB���R�Q��4u�9̓IK�����hh�i��t�ݕN��W���G���w ��Ljg(�gw��L�Ӌ�T071���oUX*�*|�� �J�&�*/T����ު U�U�T��^S}�FU3S� Ԗ�U��P�SSg�;���g�oT?�~Y��Y�L�OC�Q��_�� c�x,!k ��u�5�&���|v*�����=���9C3J3W�R�f?�q��tN �(���~���)�)�4L�1e\k����X�H�Q�G�6������E�Y��A�J'\'Gg����S�Sݧ �M=:��.�k���Dw�n��^��Lo��y���}/�T�m���G X� $� �<�5qo</���QC]�@C�a�a�ᄑ��<��F�F�i�\�$�m�mƣ&&!&KM�M�RM��)�;L;L���͢�֙5�=1�2��כ߷`ZxZ,����eI��Z�Yn�Z9Y�XUZ]�F���%ֻ�����N�N���gð�ɶ�����ۮ�m�}agbg�Ů��}�}��= ���Z~s�r:V:ޚΜ�?}����/gX���3��)�i�S��Ggg�s�󈋉K��.�>.���Ƚ�Jt�q]�z���������ۯ�6�i�ܟ�4�)�Y3s���C�Q��? ��0k߬~OCO�g��#/c/�W�װ��w��a�>�>r��>�<7�2�Y_�7��ȷ�O�o�_��C#�d�z����%g��A�[��z|!��?:�e����A���AA�������!h�쐭!��Α�i�P~���a�a��~ '���W�?�p�X�1�5w��Cs�D�D�Dޛg1O9�-J5*>�.j<�7�4�?�.fY��X�XIlK9.*�6nl������� �{�/�]py�����.,:�@L�N8��A*��%�w%� y��g"/�6ш�C\*N�H*Mz�쑼5y$�3�,幄'���L Lݛ:��v m2=:�1����qB�!M��g�g�fvˬe����n��/��k���Y- �B��TZ(�*�geWf�͉�9���+��̳�ې7�����ᒶ��KW-X潬j9�������(�x��oʿ�ܔ���Ĺd�f�f���-�[����n �ڴ �V����E�/��(ۻ��C���<��e����;?T�T�T�T6��ݵa��n��{��4���[���>ɾ�UUM�f�e�I���?�������m]�Nmq����#�׹���=TR��+�G�����w- 6 U����#pDy��� �� :�v�{���vg/jB��F�S��[b[�O�>����z�G��499�?r����C�d�&����ˮ/~�����јѡ�򗓿m|������������x31^�V���w�w��O�| (�h���SЧ��������c3-� cHRMz%������u0�`:�o�_�F5IDATx��ytUս�?�;���fN$$��@B�!�k��� -*�gE���j�O�kQ�ʠ`U�B_ �AÐ0%�Ȕ�@r3ޛ;߳��$�@��v����^�s����������$��ߢ !�W��5�̱'-٘������4�>�Rƫ$�G��0U�H}�_ _� #��k�MJ/=�9Du *ڲ�k�PQ;0��j|�*�$�^컂脛y���>�z×� B��� ��(�:$�ʔ_�6��C!B����MVx��a��ζyY����j��ƋY2�9�F�{r��r�딟F��Q��hY6��� x��[l��RC��������nt@��;P��3��� F�w��7��:�P���ɋ'���mj���Bƪ�$�Q�L��y��߷g��\���z����v5�p�v$�v櫻�`6~K �Y�X����,�:��"�o�e0II"(��,������l_�C���/4�+f�E�!q��:����u����c��樼.UE�@� �]K�W���$-�h]�JG���Dz)�`��.�A/ ���mᏣc�3jK� �!�>���$�Q�!�_�? \�C2(E �M$Z I{�U���s�ǣ���=��@Բe ޻��W�����@ �w����}�9v,)��.}*|"�s��di���,���G�]W�AC�������|<����[��ߕ������K�*�G nߴ!��‹��J�!��6(L�a)��Am��������;adf��:�xm��nh�;֦I��(���o�C~ډa?�=��s����h��E������xp����)���_���t��W�����´;8���`dz=(��@@�+,�;�`�fhl��XȌ�3,uI�~z(J����� �ZX�A����H'�À�㜦�82|��a�n�z2��I���� ���EY�h�ε<�0�q�{��C��v�˅��?"@��χ�x�WS+��N�~�KQ�J�8ل�� _.W��� !TU{�����,��"�� �;�������HI S� ��))�o�1�H�D� ���� (� ;����"Ҕ��,���� �F=���O��&�Ph>�`���y��߲ &S��P�<��'��1s�1cĂ�N����2)����@�u��tv��<�+�-&�����h��[!FSG���' � z z0��<����8d&�����E0�,��<��"!P�b4��� �{S�����X����.�囤���T��hEF��X�q0"-<����h���]�Py*����F⩍��@v�˂�Y08�� .q��a��s����(!<̫1�3�L�H��7sYW�`��qtu�\7��?�@�Y�� %Ka�x�GZ�*W�P��y�����N0r9��Z�c�%���q,��k�����!!,��>���IB��� �x1�Xu�Y���v-���ֺ��N������JP��a�2`�e5��w��u;�6�i���̃��(K���ťa��(l51��JL��O'%���]���ٖ뿉!��{*�p+f ���i8��D�ȑ���ܩ�M��V��ӟz��x�7Z�1i}�^͜���6e����x��)u�d�ˏ�V�eG�\�X1�����=n�U'�~Y����L�\,�(d!x ,�:�TW8� M`r���@��r+v�Q0��>^;˴�VbF��('����� �*O��i��j�Ӆ����^�S3&� ���{������.ی�� N;��NU%d0�Ce\r"�C�b��&t7N�7��� �ȪD�[%l�۪�$��'m?�A9tI�9>��NVL ��� 6v~M�����ъ�B���0����9$&=ώm��W�0���L���&��E���f�f��'+�9��hv��7,���M��6�D�&�^� �����8 D֮�����o��k^z��O��hP�8q�5d*��YVq3�{��������#LX4�?��jP=3�~�؅c�ݰ�̱�ұ��8̲��2�G�����j��έ�9��w7���U�X�w�85C��?!I&���o&����L���I#�^�eV�,�^t5�k� �xs˛\��U�8��."���{���ǵ�l4��ݼ�;��r"���G�/~���gJV�N糦��ZZ^k+NN�D��I$�=OE�z����|*�*{��%k���G��>�`9��b;�s���8,�qs��c��.b��q�%�>r��n�R�Ƨ�%�,�A��le#�8���}; 21�0�b�&�QX�f���q�00����$>�h�N�'�a3��A���/�r ��L5�8�GԲ'�����w;��c#�q1���h$*��V^�k��QĽ�b:i������� ����H F#�_���,��]g����Ը����`RUU���� ��"�qD���>-��ϤI�~��3�̴�����NZ[[555�eeek7�m����YF#[��Đ��R��i��L�H�r ��@��J-�Pŝ��_|�-��6y��ks�V�U~��6���w.����(��0A� 4��f��� /���C�xŊ�ٳ����z=6�����rg���mKo��o�/����a�g���?'9%9��P��斓��~���L{�� �H��{AW��B��$����v�1b�&Mb�ر��ر#�0`�`xMBp! )Z�Nmo�mZ�g�}�eذaWTVV ���'99��C5��&�&+5�bT>|8��PUUETT����&���t��~TE%��'������b�tEGG;RSS�S�N�v��ѯ��t����u1@�ޚ"f��͈���r'L�Ph��m��ƌ�74o��`�x�q�P���NRSS�X,�����l&&&��MFFA!-��`��ҷ�n��Yuu�����x<I�LB%����KԒ50�!�3�bz�����ʸKL���;---�l6[��`��������Ȏ;�����egg4��!D�T��jWujyZ��������K����%����p�^���X,6��sV=�ܖ����6�3�X ���4ҿ��A2�&����_k�;d�� �VfOIEND�B`�""" robotparser.py Copyright (C) 2000 Bastian Kleineidam You can choose between two licenses when using this package: 1) GNU GPLv2 2) PSF license for Python 2.2 The robots.txt Exclusion Protocol is implemented as specified in http://www.robotstxt.org/norobots-rfc.txt """ import collections import urllib.parse import urllib.request __all__ = ["RobotFileParser"] RequestRate = collections.namedtuple("RequestRate", "requests seconds") class RobotFileParser: """ This class provides a set of methods to read, parse and answer questions about a single robots.txt file. """ def __init__(self, url=''): self.entries = [] self.sitemaps = [] self.default_entry = None self.disallow_all = False self.allow_all = False self.set_url(url) self.last_checked = 0 def mtime(self): """Returns the time the robots.txt file was last fetched. This is useful for long-running web spiders that need to check for new robots.txt files periodically. """ return self.last_checked def modified(self): """Sets the time the robots.txt file was last fetched to the current time. """ import time self.last_checked = time.time() def set_url(self, url): """Sets the URL referring to a robots.txt file.""" self.url = url self.host, self.path = urllib.parse.urlparse(url)[1:3] def read(self): """Reads the robots.txt URL and feeds it to the parser.""" try: f = urllib.request.urlopen(self.url) except urllib.error.HTTPError as err: if err.code in (401, 403): self.disallow_all = True elif err.code >= 400 and err.code < 500: self.allow_all = True else: raw = f.read() self.parse(raw.decode("utf-8").splitlines()) def _add_entry(self, entry): if "*" in entry.useragents: # the default entry is considered last if self.default_entry is None: # the first default entry wins self.default_entry = entry else: self.entries.append(entry) def parse(self, lines): """Parse the input lines from a robots.txt file. We allow that a user-agent: line is not preceded by one or more blank lines. """ # states: # 0: start state # 1: saw user-agent line # 2: saw an allow or disallow line state = 0 entry = Entry() self.modified() for line in lines: if not line: if state == 1: entry = Entry() state = 0 elif state == 2: self._add_entry(entry) entry = Entry() state = 0 # remove optional comment and strip line i = line.find('#') if i >= 0: line = line[:i] line = line.strip() if not line: continue line = line.split(':', 1) if len(line) == 2: line[0] = line[0].strip().lower() line[1] = urllib.parse.unquote(line[1].strip()) if line[0] == "user-agent": if state == 2: self._add_entry(entry) entry = Entry() entry.useragents.append(line[1]) state = 1 elif line[0] == "disallow": if state != 0: entry.rulelines.append(RuleLine(line[1], False)) state = 2 elif line[0] == "allow": if state != 0: entry.rulelines.append(RuleLine(line[1], True)) state = 2 elif line[0] == "crawl-delay": if state != 0: # before trying to convert to int we need to make # sure that robots.txt has valid syntax otherwise # it will crash if line[1].strip().isdigit(): entry.delay = int(line[1]) state = 2 elif line[0] == "request-rate": if state != 0: numbers = line[1].split('/') # check if all values are sane if (len(numbers) == 2 and numbers[0].strip().isdigit() and numbers[1].strip().isdigit()): entry.req_rate = RequestRate(int(numbers[0]), int(numbers[1])) state = 2 elif line[0] == "sitemap": # According to http://www.sitemaps.org/protocol.html # "This directive is independent of the user-agent line, # so it doesn't matter where you place it in your file." # Therefore we do not change the state of the parser. self.sitemaps.append(line[1]) if state == 2: self._add_entry(entry) def can_fetch(self, useragent, url): """using the parsed robots.txt decide if useragent can fetch url""" if self.disallow_all: return False if self.allow_all: return True # Until the robots.txt file has been read or found not # to exist, we must assume that no url is allowable. # This prevents false positives when a user erroneously # calls can_fetch() before calling read(). if not self.last_checked: return False # search for given user agent matches # the first match counts parsed_url = urllib.parse.urlparse(urllib.parse.unquote(url)) url = urllib.parse.urlunparse(('','',parsed_url.path, parsed_url.params,parsed_url.query, parsed_url.fragment)) url = urllib.parse.quote(url) if not url: url = "/" for entry in self.entries: if entry.applies_to(useragent): return entry.allowance(url) # try the default entry last if self.default_entry: return self.default_entry.allowance(url) # agent not found ==> access granted return True def crawl_delay(self, useragent): if not self.mtime(): return None for entry in self.entries: if entry.applies_to(useragent): return entry.delay if self.default_entry: return self.default_entry.delay return None def request_rate(self, useragent): if not self.mtime(): return None for entry in self.entries: if entry.applies_to(useragent): return entry.req_rate if self.default_entry: return self.default_entry.req_rate return None def site_maps(self): if not self.sitemaps: return None return self.sitemaps def __str__(self): entries = self.entries if self.default_entry is not None: entries = entries + [self.default_entry] return '\n\n'.join(map(str, entries)) class RuleLine: """A rule line is a single "Allow:" (allowance==True) or "Disallow:" (allowance==False) followed by a path.""" def __init__(self, path, allowance): if path == '' and not allowance: # an empty value means allow all allowance = True path = urllib.parse.urlunparse(urllib.parse.urlparse(path)) self.path = urllib.parse.quote(path) self.allowance = allowance def applies_to(self, filename): return self.path == "*" or filename.startswith(self.path) def __str__(self): return ("Allow" if self.allowance else "Disallow") + ": " + self.path class Entry: """An entry has one or more user-agents and zero or more rulelines""" def __init__(self): self.useragents = [] self.rulelines = [] self.delay = None self.req_rate = None def __str__(self): ret = [] for agent in self.useragents: ret.append(f"User-agent: {agent}") if self.delay is not None: ret.append(f"Crawl-delay: {self.delay}") if self.req_rate is not None: rate = self.req_rate ret.append(f"Request-rate: {rate.requests}/{rate.seconds}") ret.extend(map(str, self.rulelines)) return '\n'.join(ret) def applies_to(self, useragent): """check if this entry applies to the specified agent""" # split the name token and make it lower case useragent = useragent.split("/")[0].lower() for agent in self.useragents: if agent == '*': # we have the catch-all agent return True agent = agent.lower() if agent in useragent: return True return False def allowance(self, filename): """Preconditions: - our agent applies to this entry - filename is URL decoded""" for line in self.rulelines: if line.applies_to(filename): return line.allowance return True