aboutcode-org · 35C4n0r · Sep 3, 2023 · Sep 3, 2023 · Sep 3, 2023 · Sep 3, 2023
diff --git a/setup.cfg b/setup.cfg
@@ -80,7 +80,7 @@ install_requires =
     gemfileparser2 >= 0.9.0
     html5lib
     importlib_metadata
-    intbitset >= 3.0.2
+    test-bitcode-alt >= 1.0.0
     jaraco.functools
     javaproperties >= 0.5
     jinja2 >= 2.7.0

diff --git a/src/licensedcode/index.py b/src/licensedcode/index.py
@@ -17,7 +17,8 @@
 import sys
 from time import time
 
-from intbitset import intbitset
+# from intbitset import intbitset
+from bitcode.bitcode import intbitset
 
 from licensedcode import SMALL_RULE
 from licensedcode import TINY_RULE
@@ -45,14 +46,14 @@
 """
 
 # Tracing flags
-TRACE = False or os.environ.get('SCANCODE_DEBUG_LICENSE', False)
-TRACE_APPROX = False
-TRACE_APPROX_CANDIDATES = False
-TRACE_APPROX_MATCHES = False
-TRACE_INDEXING = False or os.environ.get('SCANCODE_DEBUG_LICENSE_INDEX', False)
-TRACE_INDEXING_PERF = False
-TRACE_TOKEN_DOC_FREQ = False
-TRACE_SPDX_LID = False
+TRACE = True or os.environ.get('SCANCODE_DEBUG_LICENSE', False)
+TRACE_APPROX = True
+TRACE_APPROX_CANDIDATES = True
+TRACE_APPROX_MATCHES = True
+TRACE_INDEXING = True or os.environ.get('SCANCODE_DEBUG_LICENSE_INDEX', False)
+TRACE_INDEXING_PERF = True
+TRACE_TOKEN_DOC_FREQ = True
+TRACE_SPDX_LID = True
 
 
 def logger_debug(*args):

diff --git a/src/licensedcode/match_set.py b/src/licensedcode/match_set.py
@@ -12,7 +12,8 @@
 from functools import partial
 from itertools import groupby
 
-from intbitset import intbitset
+# from intbitset import intbitset
+from bitcode.bitcode import intbitset
 
 from licensedcode.tokenize import ngrams
 

diff --git a/src/licensedcode/query.py b/src/licensedcode/query.py
@@ -15,7 +15,8 @@
 from functools import partial
 from itertools import chain
 
-from intbitset import intbitset
+# from intbitset import intbitset
+from bitcode.bitcode import intbitset
 
 import typecode
 

diff --git a/src/licensedcode/spans.py b/src/licensedcode/spans.py
@@ -31,7 +31,8 @@
 from itertools import count
 from itertools import groupby
 
-from intbitset import intbitset
+# from intbitset import intbitset
+from bitcode.bitcode import intbitset
 
 """
 Ranges and intervals of integers using bitmaps.

diff --git a/tests/fails/data/debian/stable_firmware-realtek.copyright b/tests/fails/data/debian/stable_firmware-realtek.copyright
@@ -0,0 +1,53 @@
+Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: linux-firmware
+Upstream-Contact: linux-firmware@kernel.org
+Source: https://git.kernel.org/cgit/linux/kernel/git/firmware/linux-firmware.git
+
+Files: rtl_nic/*
+Copyright: 2010-2013, Realtek Semiconductor Corporation 
+License: Binary redistribution (Realtek permissive)
+ Permission is hereby granted for the distribution of this firmware
+ data in hexadecimal or equivalent format, provided this copyright
+ notice is accompanying it.
+
+Files: RTL8192E/*, RTL8192SU/*, rtlwifi/*
+Copyright: 2009-2010, Realtek Semiconductor Corporation 
+License: Binary redistribution (Realtek restrictive)
+ All rights reserved.
+ .
+ Redistribution.  Redistribution and use in binary form, without 
+ modification, are permitted provided that the following conditions are 
+ met:
+ .
+ * Redistributions must reproduce the above copyright notice and the 
+   following disclaimer in the documentation and/or other materials 
+   provided with the distribution. 
+ * Neither the name of Realtek Semiconductor Corporation nor the names of its
+   suppliers may be used to endorse or promote products derived from this
+   software without specific prior written permission. 
+ * No reverse engineering, decompilation, or disassembly of this software 
+   is permitted.
+ .
+ Limited patent license. Realtek Semiconductor Corporation grants a world-wide, 
+ royalty-free, non-exclusive license under patents it now or hereafter 
+ owns or controls to make, have made, use, import, offer to sell and 
+ sell ("Utilize") this software, but solely to the extent that any 
+ such patent is necessary to Utilize the software alone, or in 
+ combination with an operating system licensed under an approved Open 
+ Source license as listed by the Open Source Initiative at 
+ http://opensource.org/licenses.  The patent license shall not apply to 
+ any other combinations which include this software.  No hardware per 
+ se is licensed hereunder.
+ .
+ DISCLAIMER.  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND 
+ CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, 
+ BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND 
+ FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 
+ COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 
+ BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS 
+ OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 
+ ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR 
+ TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 
+ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 
+ DAMAGE.
diff --git a/tests/fails/data/debian/stable_firmware-realtek.copyright-detailed.expected.yml b/tests/fails/data/debian/stable_firmware-realtek.copyright-detailed.expected.yml
@@ -0,0 +1,79 @@
+declared_license:
+  - Binary redistribution (Realtek permissive)
+  - Binary redistribution (Realtek restrictive)
+declared_license_expression: other-permissive AND intel
+declared_license_expression_spdx: LicenseRef-scancode-other-permissive AND LicenseRef-scancode-intel
+other_license_expression:
+other_license_expression_spdx:
+license_detections:
+  - license_expression: other-permissive
+    matches:
+      - score: '100.0'
+        start_line: 9
+        end_line: 11
+        matched_length: 23
+        match_coverage: '100.0'
+        matcher: 1-hash
+        license_expression: other-permissive
+        rule_identifier: other-permissive_66.RULE
+        rule_relevance: 100
+        rule_url: https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/other-permissive_66.RULE
+        matched_text: |
+          Permission is hereby granted for the distribution of this firmware
+          data in hexadecimal or equivalent format, provided this copyright
+          notice is accompanying it.
+    identifier: other_permissive-16dc1659-320f-55c9-0ddb-4d46c5526cbb
+  - license_expression: intel
+    matches:
+      - score: '93.72'
+        start_line: 18
+        end_line: 53
+        matched_length: 293
+        match_coverage: '100.0'
+        matcher: 3-seq
+        license_expression: intel
+        rule_identifier: intel_1.RULE
+        rule_relevance: 95
+        rule_url: https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/intel_1.RULE
+        matched_text: |
+          Redistribution.  Redistribution and use in binary form, without
+          modification, are permitted provided that the following conditions are
+          met:
+
+          * Redistributions must reproduce the above copyright notice and the
+            following disclaimer in the documentation and/or other materials
+            provided with the distribution.
+          * Neither the name of Realtek Semiconductor Corporation nor the names of its
+            suppliers may be used to endorse or promote products derived from this
+            software without specific prior written permission.
+          * No reverse engineering, decompilation, or disassembly of this software
+            is permitted.
+
+          Limited patent license. Realtek Semiconductor Corporation grants a world-wide,
+          royalty-free, non-exclusive license under patents it now or hereafter
+          owns or controls to make, have made, use, import, offer to sell and
+          sell ("Utilize") this software, but solely to the extent that any
+          such patent is necessary to Utilize the software alone, or in
+          combination with an operating system licensed under an approved Open
+          Source license as listed by the Open Source Initiative at
+          http://opensource.org/licenses.  The patent license shall not apply to
+          any other combinations which include this software.  No hardware per
+          se is licensed hereunder.
+
+          DISCLAIMER.  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+          CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,
+          BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+          FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+          COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+          INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+          BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+          OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+          ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
+          TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+          USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+          DAMAGE.
+    identifier: intel-066e3a31-92c6-4f54-d106-62df186d8643
+other_license_detections: []
+copyright: |
+  2010-2013, Realtek Semiconductor Corporation
+  2009-2010, Realtek Semiconductor Corporation
diff --git a/tests/fails/data/licenses/free-unknown.LICENSE b/tests/fails/data/licenses/free-unknown.LICENSE
@@ -0,0 +1,11 @@
+---
+key: free-unknown
+short_name: Free unknown
+name: Free unknown license detected but not recognized
+category: Unstated License
+owner: Unspecified
+notes: This case applies to software with a notice that refers in a non-specific manner to a
+    free or open-source license, but where it is not possible to determine that specific license.
+is_unknown: yes
+spdx_license_key: LicenseRef-scancode-free-unknown
+---
diff --git a/tests/fails/data/licenses/intel.LICENSE b/tests/fails/data/licenses/intel.LICENSE
@@ -0,0 +1,55 @@
+---
+key: intel
+short_name: Intel Limited Patent License
+name: Intel Limited Patent License
+category: Proprietary Free
+owner: Intel Corporation
+homepage_url: https://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git/tree/LICENCE.IntcSST2
+spdx_license_key: LicenseRef-scancode-intel
+text_urls:
+    - https://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git/tree/LICENCE.adsp_sst
+other_urls:
+    - http://web.archive.org/web/20060924155349/http://www.mepis.org/node/10358
+ignorable_urls:
+    - http://opensource.org/licenses
+---
+
+Redistribution.
+Redistribution and use in binary form, without modification, are
+permitted provided that the following conditions are met:
+
+Redistributions must reproduce the above copyright notice and the
+following disclaimer in the documentation and/or other materials
+provided with the distribution.
+
+Neither the name of Intel Corporation
+nor the names of its suppliers may be used to endorse or promote
+products derived from this software without specific prior written
+permission.
+
+No reverse engineering, decompilation, or disassembly of this
+software is permitted.
+
+Limited patent license. Intel Corporation grants a world-wide,
+royalty-free, non-exclusive license under patents it now or hereafter
+owns or controls to make, have made, use, import, offer to sell and
+sell ("Utilize") this software, but solely to the extent that any
+such patent is necessary to Utilize the software alone, or in
+combination with an operating system licensed under an approved Open
+Source license as listed by the Open Source Initiative at
+http://opensource.org/licenses.  The patent license shall not apply to
+any other combinations which include this software.  No hardware per
+se is licensed hereunder.
+
+DISCLAIMER.  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND 
+CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, 
+BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND 
+FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 
+COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 
+BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS 
+OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR 
+TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 
+DAMAGE.
diff --git a/tests/fails/data/licenses/other-permissive.LICENSE b/tests/fails/data/licenses/other-permissive.LICENSE
@@ -0,0 +1,17 @@
+---
+key: other-permissive
+short_name: Other Permissive Licenses
+name: Other Permissive Licenses
+category: Permissive
+owner: nexB
+notes: |
+    this is a catch all and ellipsis to deal with some cases when a large
+    number of ancillary yet similar licenses may be reported as one.
+is_generic: yes
+spdx_license_key: LicenseRef-scancode-other-permissive
+other_spdx_license_keys:
+    - LicenseRef-Fedora-UltraPermissive
+---
+
+This component contains multiple third-party subcomponents licensed
+under permissive licenses in the style of MIT, BSD, X11, and/or Apache.
diff --git a/tests/fails/data/licenses/unknown-license-reference.LICENSE b/tests/fails/data/licenses/unknown-license-reference.LICENSE
@@ -0,0 +1,12 @@
+---
+key: unknown-license-reference
+short_name: Unknown License reference
+name: Unknown License file reference
+category: Unstated License
+owner: Unspecified
+notes: This applies to the case of a file with no clear license, which may be referenced via
+    URL or text such as "See license in..." or "This file is licensed under...", but where the
+    reference cannot be resolved to a specific named, public license.
+is_unknown: yes
+spdx_license_key: LicenseRef-scancode-unknown-license-reference
+---
diff --git a/tests/fails/data/rules/free-unknown_88.RULE b/tests/fails/data/rules/free-unknown_88.RULE
@@ -0,0 +1,7 @@
+---
+license_expression: free-unknown
+is_license_clue: yes
+relevance: 50
+---
+
+open source license
diff --git a/tests/fails/data/rules/free-unknown_96.RULE b/tests/fails/data/rules/free-unknown_96.RULE
@@ -0,0 +1,9 @@
+---
+license_expression: free-unknown
+is_license_reference: yes
+relevance: 100
+ignorable_urls:
+    - http://opensource.org/licenses/
+---
+
+http://opensource.org/licenses/
diff --git a/tests/fails/data/rules/intel_1.RULE b/tests/fails/data/rules/intel_1.RULE
@@ -0,0 +1,45 @@
+---
+license_expression: intel
+is_license_text: yes
+relevance: 95
+notes: intel for Realtek
+ignorable_urls:
+    - http://opensource.org/licenses
+---
+
+Redistribution.  Redistribution and use in binary form, without 
+modification, are permitted provided that the following conditions are 
+met:
+
+* Redistributions must reproduce the above copyright notice and the 
+  following disclaimer in the documentation and/or other materials 
+  provided with the distribution. 
+* Neither the name of Corporation nor the names of its
+  suppliers may be used to endorse or promote products derived from this
+  software without specific prior written permission. 
+* No reverse engineering, decompilation, or disassembly of this software 
+  is permitted.
+
+Limited patent license. Corporation grants a world-wide, 
+royalty-free, non-exclusive license under patents it now or hereafter 
+owns or controls to make, have made, use, import, offer to sell and 
+sell ("Utilize") this software, but solely to the extent that any 
+such patent is necessary to Utilize the software alone, or in 
+combination with an operating system licensed under an approved Open 
+Source license as listed by the Open Source Initiative at 
+http://opensource.org/licenses.  The patent license shall not apply to 
+any other combinations which include this software.  No hardware per 
+se is licensed hereunder.
+
+DISCLAIMER.  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND 
+CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, 
+BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND 
+FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 
+COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 
+BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS 
+OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR 
+TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 
+DAMAGE.
diff --git a/tests/fails/data/rules/license-intro_2.RULE b/tests/fails/data/rules/license-intro_2.RULE
@@ -0,0 +1,7 @@
+---
+license_expression: unknown-license-reference
+is_license_intro: yes
+relevance: 50
+---
+
+Licensed under
diff --git a/tests/fails/data/rules/other-permissive_66.RULE b/tests/fails/data/rules/other-permissive_66.RULE
@@ -0,0 +1,9 @@
+---
+license_expression: other-permissive
+is_license_text: yes
+relevance: 100
+---
+
+Permission is hereby granted for the distribution of this firmware data
+  in hexadecimal or equivalent format, provided this copyright notice is
+  accompanying it.