Skip to content

Remove GPL/LGPL-derived string handling and transliteration #42

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
jfstenuit wants to merge 1 commit into from
Closed

Remove GPL/LGPL-derived string handling and transliteration #42

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,14 @@ For PHP^7.3 and PHP^8.0 use 3.x (backward compatible, but phpunit^9 don't work w
composer require wamania/php-stemmer "^3.0"
```

For PHP^8.4 use 4.x (avoid deprecated by switching from voku utf8 to [joomla/string](https://github.com/joomla-framework/string) )
For PHP^8.4 use 4.x. Earlier versions relied on [voku/portable-utf8](https://github.com/voku/portable-utf8)
for UTF-8-safe string handling, back when `ext-mbstring` couldn't be assumed present.
It was briefly replaced by [joomla/string](https://github.com/joomla-framework/string) to avoid
`voku/portable-utf8`'s deprecation notices on PHP 8.4, but that package is GPL-2.0-or-later
licensed, which is incompatible with this project's MIT license, and for the handful of
functions used here it was only ever a thin wrapper around PHP's built-in `mbstring` extension.
4.x calls `mb_*` functions directly instead and requires `ext-mbstring` (bundled/enabled in
virtually all PHP installations).
```
composer require wamania/php-stemmer "^4.0"
```
Expand Down
2 changes: 1 addition & 1 deletion composer.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
],
"require": {
"php": ">=7.3",
"joomla/string": ">=2.0.1"
"ext-mbstring": "*"
},
"require-dev":{
"phpunit/phpunit": "^9.0"
Expand Down
16 changes: 7 additions & 9 deletions src/Stemmer/Catalan.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@

namespace Wamania\Snowball\Stemmer;

use Joomla\String\StringHelper;

/**
*
* @link http://snowball.tartarus.org/algorithms/catalan/stemmer.html
Expand Down Expand Up @@ -86,7 +84,7 @@ class Catalan extends Stem
*/
public function stem($word)
{
$this->word = StringHelper::strtolower($word);
$this->word = mb_strtolower($word);

// Catalan stemmer does not use Rv
$this->r1();
Expand Down Expand Up @@ -122,7 +120,7 @@ private function step0()
{
if (($position = $this->search(static::$attached_pronoun)) !== false) {
if ($this->inR1($position)) {
$this->word = StringHelper::substr($this->word, 0, $position);
$this->word = mb_substr($this->word, 0, $position);
return true;
}
}
Expand All @@ -141,7 +139,7 @@ private function step1a()
// delete if in R2
if (($position = $this->search(['acions', 'ada', 'ades'])) !== false) {
if ($this->inR2($position)) {
$this->word = StringHelper::substr($this->word, 0, $position);
$this->word = mb_substr($this->word, 0, $position);
}
return true;
}
Expand All @@ -161,7 +159,7 @@ private function step1a()
// delete if in R1
if (($position = $this->search(self::$standard_suffix_1a)) !== false) {
if ($this->inR1($position)) {
$this->word = StringHelper::substr($this->word, 0, $position);
$this->word = mb_substr($this->word, 0, $position);
}
return true;
}
Expand Down Expand Up @@ -236,7 +234,7 @@ private function step1b()
// delete if in R1
if (($position = $this->search(static::$verb_suffixes)) !== false) {
if ($this->inR1($position)) {
$this->word = StringHelper::substr($this->word, 0, $position);
$this->word = mb_substr($this->word, 0, $position);
}
return true;
}
Expand All @@ -246,7 +244,7 @@ private function step1b()
// delete if in R2
if (($position = $this->search(['ando'])) !== false) {
if ($this->inR2($position)) {
$this->word = StringHelper::substr($this->word, 0, $position);
$this->word = mb_substr($this->word, 0, $position);
}
return true;
}
Expand All @@ -265,7 +263,7 @@ private function step2()
// delete if in R1
if (($position = $this->search(static::$residual_suffixes)) !== false) {
if ($this->inR1($position)) {
$this->word = StringHelper::substr($this->word, 0, $position);
$this->word = mb_substr($this->word, 0, $position);
}
return true;
}
Expand Down
28 changes: 13 additions & 15 deletions src/Stemmer/Danish.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@

namespace Wamania\Snowball\Stemmer;

use Joomla\String\StringHelper;

/**
*
* @link http://snowball.tartarus.org/algorithms/danish/stemmer.html
Expand All @@ -22,15 +20,15 @@ class Danish extends Stem
*/
public function stem($word): string
{
$this->word = StringHelper::strtolower($word);
$this->word = mb_strtolower($word);

// R2 is not used: R1 is defined in the same way as in the German stemmer
$this->r1();

// then R1 is adjusted so that the region before it contains at least 3 letters.
if ($this->r1Index < 3) {
$this->r1Index = 3;
$this->r1 = StringHelper::substr($this->word, 3);
$this->r1 = mb_substr($this->word, 3);
}

// Do each of steps 1, 2 3 and 4.
Expand All @@ -51,7 +49,7 @@ public function stem($word): string
*/
private function hasValidSEnding($word)
{
$lastLetter = StringHelper::substr($word, -1, 1);
$lastLetter = mb_substr($word, -1, 1);
return in_array($lastLetter, array('a', 'b', 'c', 'd', 'f', 'g', 'h', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'r', 't', 'v', 'y', 'z', 'å'));
}

Expand All @@ -69,14 +67,14 @@ private function step1()
'erens', 'ered', 'ende', 'erne', 'eres', 'eren', 'eret', 'erer', 'enes', 'heds',
'ens', 'ene', 'ere', 'ers', 'ets', 'hed', 'es', 'et', 'er', 'en', 'e'
))) !== false) {
$this->word = StringHelper::substr($this->word, 0, $position);
$this->word = mb_substr($this->word, 0, $position);
return true;
}

// s
// delete if preceded by a valid s-ending
if ( ($position = $this->searchIfInR1(array('s'))) !== false) {
$word = StringHelper::substr($this->word, 0, $position);
$word = mb_substr($this->word, 0, $position);
if ($this->hasValidSEnding($word)) {
$this->word = $word;
}
Expand All @@ -92,7 +90,7 @@ private function step1()
private function step2()
{
if ($this->searchIfInR1(array('gd', 'dt', 'gt', 'kt')) !== false) {
$this->word = StringHelper::substr($this->word, 0, -1);
$this->word = mb_substr($this->word, 0, -1);
}
}

Expand All @@ -103,22 +101,22 @@ private function step3()
{
// If the word ends igst, remove the final st.
if ($this->search(array('igst')) !== false) {
$this->word = StringHelper::substr($this->word, 0, -2);
$this->word = mb_substr($this->word, 0, -2);
}

// Search for the longest among the following suffixes in R1, and perform the action indicated.
// ig lig elig els
// delete, and then repeat step 2
if ( ($position = $this->searchIfInR1(array('elig', 'lig', 'ig', 'els'))) !== false) {
$this->word = StringHelper::substr($this->word, 0, $position);
$this->word = mb_substr($this->word, 0, $position);
$this->step2();
return true;
}

// løst
// replace with løs
if ($this->searchIfInR1(array('løst')) !== false) {
$this->word = StringHelper::substr($this->word, 0, -1);
$this->word = mb_substr($this->word, 0, -1);
}
}

Expand All @@ -128,19 +126,19 @@ private function step3()
*/
private function step4()
{
$length = StringHelper::strlen($this->word);
$length = mb_strlen($this->word);
if (!$this->inR1(($length-1))) {
return false;
}

$lastLetter = StringHelper::substr($this->word, -1, 1);
$lastLetter = mb_substr($this->word, -1, 1);
if (in_array($lastLetter, self::$vowels)) {
return false;
}
$beforeLastLetter = StringHelper::substr($this->word, -2, 1);
$beforeLastLetter = mb_substr($this->word, -2, 1);

if ($lastLetter == $beforeLastLetter) {
$this->word = StringHelper::substr($this->word, 0, -1);
$this->word = mb_substr($this->word, 0, -1);
}
return true;
}
Expand Down
Loading