diff --git a/.cursor/rules/article-reference-sps.mdc b/.cursor/rules/article-reference-sps.mdc new file mode 100644 index 0000000..d97c09a --- /dev/null +++ b/.cursor/rules/article-reference-sps.mdc @@ -0,0 +1,64 @@ +--- +description: Marcação SPS de referências bibliográficas (ref-list / element-citation) +globs: reference/**/* +alwaysApply: true +--- + +# Lista de referências (SciELO SPS) + +Ao marcar, gerar ou validar referências em XML/JSON neste projeto, seguir Critérios SciELO Brasil / SPS. + +## Estrutura obrigatória + +- `` em `` (obrigatório em documentos indexáveis, exceto errata, retratação, adendo, preocupação, parecer). +- Cada `` contém **obrigatoriamente** `` e ``. +- `` exige `@publication-type`. + +| Tag | Função | +|---|---| +| `mixed-citation` | Texto da referência como apresentado (espaços, pontuação). Só formatação: `bold`, `italic`, `sup`, `sub`. | +| `element-citation` | Marcação estruturada para métricas. Sem pontuação entre elementos filhos. | + +## `@publication-type` permitidos + +`book` · `confproc` · `data` · `database` · `journal` · `legal-doc` · `letter` · `newspaper` · `patent` · `preprint` · `report` · `software` · `thesis` · `webpage` · `other` + +## Títulos por tipo + +| Tipo | Tag de título | +|---|---| +| `journal` | `article-title` + `source` (periódico) | +| `book` (obra) | `source` | +| `book` (capítulo) | `part-title` (capítulo) + `source` (livro). **Não** usar `chapter-title`. | +| `data` | `data-title` + `source` | +| `confproc` | `conf-name` (+ `conf-loc`, `conf-date`, `conf-num`, `conf-sponsor` quando houver) | +| `webpage` | `source` | + +## Regras de marcação + +- Preferir `source` + `year` quando a norma bibliográfica for completa. +- Autoria: `person-group` com `@person-group-type` (`author`, `editor`, `translator`, `compiler`). Nome único → só `surname`. Instituição → `collab` (com `author`). +- `fpage` implica `lpage`. +- `size` → `@units="pages"`. +- `date-in-citation` → `@content-type="access-date"`. +- DOI: `pub-id` com `@pub-id-type="doi"`. +- URL: `ext-link` com `@ext-link-type="uri"` (ou `doi`). **No máximo um** `ext-link` em `element-citation` e em `mixed-citation`. +- Sem elemento específico → `comment`. Proibido `…`; permitido texto + `ext-link` dentro de `comment`. +- Não envolver o texto inteiro de um elemento só com `italic`/`bold` em `element-citation`. + +## Fixtures e testes do app `reference` + +- Corpus de entrada: `reference/fixtures/references.txt` (uma ref por linha) → constante `REFERENCES`. +- Golden JATS: `reference/fixtures/references.xml` (`ref-list`). +- Contagens de `.txt` e `.xml` devem coincidir (`test_eval_corpus_aligned`). + +## Código local + +- JSON → XML: `reference/data_utils.get_xml` (campo `reftype` = `@publication-type`). +- Ao evoluir o gerador, alinhar tags ao SPS (`part-title` para capítulos, tipos `data`/`software`/`legal-doc`, etc.). + +## Referências externas + +- Critérios SciELO Brasil 5.2.8.1 (XML / SPS) +- Guia de citação de dados de pesquisa +- Sample PubMed Central Citations diff --git a/.cursor/rules/model.mdc b/.cursor/rules/model.mdc new file mode 100644 index 0000000..3c61ec9 --- /dev/null +++ b/.cursor/rules/model.mdc @@ -0,0 +1,9 @@ +--- +description: Sobre modelo de IA +alwaysApply: true +--- + +# Não utilizar provedores de IA + +- Não posso utilizar APIs de IA para marcação. +- Não é para utilizar API de ia nesse projeto somente modelo local \ No newline at end of file diff --git a/.envs.example/.local/.django b/.envs.example/.local/.django index 1f8e081..32b0242 100644 --- a/.envs.example/.local/.django +++ b/.envs.example/.local/.django @@ -8,3 +8,13 @@ REDIS_URL=redis://redis:6379/0 # Celery # ------------------------------------------------------------------------------ + +# Reference (Llama via HTTP — serviço ollama no local.yml) +# ------------------------------------------------------------------------------ +REFERENCE_ENABLED=true +REFERENCE_URL=http://ollama:11434 +REFERENCE_MODEL=llama3.2:3b +# REFERENCE_TOKEN= +# REFERENCE_TIMEOUT=300 +# REFERENCE_BATCH_SIZE=10 +# REFERENCE_NUM_CTX=8192 diff --git a/.envs.example/.production/.django b/.envs.example/.production/.django index 6c842eb..c03a932 100644 --- a/.envs.example/.production/.django +++ b/.envs.example/.production/.django @@ -19,3 +19,13 @@ DJANGO_ALLOWED_HOSTS=* # Sentry # ------------------------------------------------------------------------------ SENTRY_DSN= + +# Reference (Llama via HTTP — Ollama-compatible API) +# ------------------------------------------------------------------------------ +REFERENCE_ENABLED=true +REFERENCE_URL= +REFERENCE_MODEL=llama3.2:3b +# REFERENCE_TOKEN= +# REFERENCE_TIMEOUT=300 +# REFERENCE_BATCH_SIZE=10 +# REFERENCE_NUM_CTX=8192 diff --git a/.gitignore b/.gitignore index dc41aef..11eade9 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ venv/ .envs .envs.local *.envs +.token # Logs *.log diff --git a/Makefile b/Makefile index 214d367..b7c70a5 100755 --- a/Makefile +++ b/Makefile @@ -63,14 +63,19 @@ django_createsuperuser: ## Create a superuser django_bash: ## Open bash in django container docker compose -f $(COMPOSE_FILE) run --rm django bash -test: ## Run tests (pytest default) - docker compose -f $(COMPOSE_FILE) run --rm django pytest --reuse-db +test: ## Run tests (pytest default, excludes llama eval) + docker compose -f $(COMPOSE_FILE) run --rm django pytest --reuse-db -m "not llama" + +test-llama: ## Run Llama reference eval tests (requires Ollama) + docker compose -f $(COMPOSE_FILE) run --rm django pytest --reuse-db -m llama reference/tests/test_references.py test-fast: ## Run tests (pytest failfast) - docker compose -f $(COMPOSE_FILE) run --rm django pytest -x --reuse-db + docker compose -f $(COMPOSE_FILE) run --rm django pytest -x --reuse-db -m "not llama" -test-cov: ## Run tests with coverage - docker compose -f $(COMPOSE_FILE) run --rm django pytest --reuse-db --cov=manuscripts --cov-report=term-missing manuscripts/tests +test-cov: ## Run tests with coverage (reference, fail under 100%) + docker compose -f $(COMPOSE_FILE) run --rm django pytest --reuse-db -m "not llama" \ + --cov=reference --cov-report=term-missing --cov-fail-under=100 \ + reference/tests test-fresh: ## Recreate test database and run pytest docker compose -f $(COMPOSE_FILE) run --rm django pytest --create-db @@ -120,6 +125,33 @@ restore_data: ## Restore database from backup/latest.sql volume_down: ## Remove all volumes docker compose -f $(COMPOSE_FILE) down -v +############################################ +## ollama / reference +############################################ + +REFERENCE_MODEL ?= llama3.2:3b + +ollama_pull: ## Pull Llama model into local ollama container + docker compose -f $(COMPOSE_FILE) exec ollama ollama pull $(REFERENCE_MODEL) + +############################################ +## JWT +############################################ + +JWT_USERNAME ?= +JWT_PASSWORD ?= + +bearer_token: ## eval "$$(make bearer_token JWT_USERNAME=u JWT_PASSWORD=p)" then curl -H "Authorization: Bearer $$TOKEN" + @test -n "$(JWT_USERNAME)" && test -n "$(JWT_PASSWORD)" || (echo 'Usage: eval "$$(make bearer_token JWT_USERNAME=user JWT_PASSWORD=pass)"' >&2 && exit 1) + @TOKEN=$$(docker compose -f $(COMPOSE_FILE) run --rm -T \ + -e JWT_USERNAME=$(JWT_USERNAME) \ + -e JWT_PASSWORD=$(JWT_PASSWORD) \ + django python manage.py shell -c "from django.contrib.auth import authenticate; from rest_framework_simplejwt.tokens import RefreshToken; import os, sys; user = authenticate(username=os.environ['JWT_USERNAME'], password=os.environ['JWT_PASSWORD']); sys.exit(1) if not user else print(RefreshToken.for_user(user).access_token)" \ + 2>/dev/null | tr -d '\r' | grep -Eo 'eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+' | tail -n 1); \ + if [ -z "$$TOKEN" ]; then echo "Failed to obtain JWT (check JWT_USERNAME/JWT_PASSWORD)" >&2; exit 1; fi; \ + printf '%s\n' "$$TOKEN" > .token; \ + printf "export TOKEN='%s'\n" "$$TOKEN" + ############################################ ## Cleanup ############################################ diff --git a/config/api_router.py b/config/api_router.py new file mode 100644 index 0000000..341dcad --- /dev/null +++ b/config/api_router.py @@ -0,0 +1,13 @@ +from django.conf import settings +from rest_framework.routers import DefaultRouter, SimpleRouter + +from reference.api.v1.views import ReferenceViewSet + +if settings.DEBUG: + router = DefaultRouter() +else: + router = SimpleRouter() + +router.register("reference", ReferenceViewSet, basename="reference") + +urlpatterns = router.urls diff --git a/config/menu.py b/config/menu.py index 605aeb2..0aba9b2 100644 --- a/config/menu.py +++ b/config/menu.py @@ -1,5 +1,6 @@ WAGTAIL_MENU_GROUPS_ORDER = [ "celery_wagtail", + "reference", ] diff --git a/config/settings/base.py b/config/settings/base.py index 89469f0..7fa8bd4 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -13,6 +13,7 @@ # Build paths inside the project like this: os.path.join(BASE_DIR, ...) import os +from datetime import timedelta from pathlib import Path import environ @@ -40,7 +41,7 @@ "core.home", "wagtail.contrib.forms", "wagtail.contrib.redirects", - 'wagtail.contrib.settings', + "wagtail.contrib.settings", "wagtail_modeladmin", "wagtail.embeds", "wagtail.sites", @@ -63,13 +64,15 @@ "django.contrib.sessions", "django.contrib.messages", "django.contrib.staticfiles", - "django_celery_results" + "django_celery_results", ] THIRD_PARTY_APPS = [ "compressor", "wagtailautocomplete", "django_celery_beat", + "rest_framework", + "wagtail_json_widget", ] LOCAL_APPS = [ @@ -77,6 +80,7 @@ "core", "core_settings", "xml_manager", + "reference", ] INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS + WAGTAIL @@ -85,7 +89,7 @@ "django.contrib.sessions.middleware.SessionMiddleware", "django.middleware.common.CommonMiddleware", "django.middleware.csrf.CsrfViewMiddleware", - 'django.middleware.locale.LocaleMiddleware', + "django.middleware.locale.LocaleMiddleware", "django.contrib.auth.middleware.AuthenticationMiddleware", "django.contrib.messages.middleware.MessageMiddleware", "django.middleware.clickjacking.XFrameOptionsMiddleware", @@ -106,7 +110,7 @@ "django.template.context_processors.request", "django.contrib.auth.context_processors.auth", "django.contrib.messages.context_processors.messages", - 'wagtail.contrib.settings.context_processors.settings', + "wagtail.contrib.settings.context_processors.settings", ], }, }, @@ -145,13 +149,13 @@ LANGUAGE_CODE = "en" LANGUAGES = [ - ('pt-br', 'Português (Brasil)'), - ('es', 'Español'), - ('en', 'English'), + ("pt-br", "Português (Brasil)"), + ("es", "Español"), + ("en", "English"), ] LOCALE_PATHS = [ - os.path.join(BASE_DIR, 'locale'), + os.path.join(BASE_DIR, "locale"), ] TIME_ZONE = "UTC" @@ -239,10 +243,22 @@ # This can be omitted to allow all files, but note that this may present a security risk # if untrusted users are allowed to upload files - # see https://docs.wagtail.org/en/stable/advanced_topics/deploying.html#user-uploaded-files -WAGTAILDOCS_EXTENSIONS = ['csv', 'docx', 'json', 'key', 'odt', 'pdf', 'pptx', 'rtf', 'txt', 'xlsx', 'zip'] +WAGTAILDOCS_EXTENSIONS = [ + "csv", + "docx", + "json", + "key", + "odt", + "pdf", + "pptx", + "rtf", + "txt", + "xlsx", + "zip", +] # https://docs.djangoproject.com/en/dev/ref/settings/#auth-user-model -AUTH_USER_MODEL = 'users.CustomUser' +AUTH_USER_MODEL = "users.CustomUser" # Celery # ------------------------------------------------------------------------------ @@ -269,13 +285,13 @@ CELERY_BEAT_SCHEDULER = "django_celery_beat.schedulers:DatabaseScheduler" # http://docs.celeryproject.org/en/latest/userguide/configuration.html DJANGO_CELERY_BEAT_TZ_AWARE = False -#CELERY PROMETHEUS DASHBOARD +# CELERY PROMETHEUS DASHBOARD # https://docs.celeryq.dev/en/stable/userguide/configuration.html#worker-send-task-events CELERY_WORKER_SEND_TASK_EVENTS = True # https://docs.celeryq.dev/en/stable/userguide/configuration.html#std-setting-task_send_sent_event CELERY_SEND_TASK_SENT_EVENT = True CELERYD_SEND_EVENTS = True -CE_BUCKETS=1,2.5,5,10,30,60,300,600,900,1800 +CE_BUCKETS = 1, 2.5, 5, 10, 30, 60, 300, 600, 900, 1800 # Celery Results # ------------------------------------------------------------------------------ @@ -285,3 +301,27 @@ CELERY_RESULT_EXTENDED = True DATA_UPLOAD_MAX_NUMBER_FIELDS = 10000 +SILENCED_SYSTEM_CHECKS = ["treebeard.E001"] + +REST_FRAMEWORK = { + "DEFAULT_PAGINATION_CLASS": "rest_framework.pagination.PageNumberPagination", + "PAGE_SIZE": env.int("DRF_PAGE_SIZE", default=10), + "DEFAULT_AUTHENTICATION_CLASSES": ( + "rest_framework_simplejwt.authentication.JWTAuthentication", + "rest_framework.authentication.SessionAuthentication", + ), +} + +SIMPLE_JWT = { + "AUTH_HEADER_TYPES": ("Bearer",), + "ACCESS_TOKEN_LIFETIME": timedelta(minutes=60), + "REFRESH_TOKEN_LIFETIME": timedelta(days=1), +} + +REFERENCE_ENABLED = env.bool("REFERENCE_ENABLED", default=True) +REFERENCE_URL = env("REFERENCE_URL", default="") +REFERENCE_MODEL = env("REFERENCE_MODEL", default="llama3.2:3b") +REFERENCE_TIMEOUT = env.int("REFERENCE_TIMEOUT", default=300) +REFERENCE_TOKEN = env("REFERENCE_TOKEN", default="") +REFERENCE_BATCH_SIZE = env.int("REFERENCE_BATCH_SIZE", default=10) +REFERENCE_NUM_CTX = env.int("REFERENCE_NUM_CTX", default=8192) diff --git a/config/urls.py b/config/urls.py index 88372e2..c766adc 100644 --- a/config/urls.py +++ b/config/urls.py @@ -3,14 +3,24 @@ from django.conf.urls.static import static from django.contrib import admin from django.urls import include, path +from rest_framework_simplejwt.views import TokenObtainPairView, TokenRefreshView from wagtail import urls as wagtail_urls from wagtail.admin import urls as wagtailadmin_urls from wagtail.documents import urls as wagtaildocs_urls +from config import api_router + urlpatterns = [ path("django-admin/", admin.site.urls), path("admin/", include(wagtailadmin_urls)), path("documents/", include(wagtaildocs_urls)), + path("api/v1/auth/token/", TokenObtainPairView.as_view(), name="token_obtain_pair"), + path( + "api/v1/auth/token/refresh/", + TokenRefreshView.as_view(), + name="token_refresh", + ), + path("api/v1/", include(api_router)), path("i18n/", include("django.conf.urls.i18n")), ] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) diff --git a/core/wagtail_hooks.py b/core/wagtail_hooks.py index 8bb4b16..00794ed 100644 --- a/core/wagtail_hooks.py +++ b/core/wagtail_hooks.py @@ -21,13 +21,6 @@ def ensure_image_title(sender, instance, **kwargs): pre_save.connect(ensure_image_title, sender=get_image_model()) -@hooks.register("construct_main_menu") -def keep_only_sps_validation_menu(request, menu_items): - menu_items[:] = [ - item for item in menu_items if item.name == "sps_package_validation" - ] - - @hooks.register("construct_help_menu") def replace_help_menu_items(request, help_menu_items): help_menu_items[:] = [ diff --git a/local.yml b/local.yml index a4acd9a..3b68887 100644 --- a/local.yml +++ b/local.yml @@ -11,6 +11,7 @@ services: - redis - postgres - mailhog + - ollama volumes: - .:/app:z # - ../packtools:/packtools:z @@ -21,6 +22,26 @@ services: - "8000:8000" command: /start + ollama: + image: ollama/ollama:latest + container_name: scielo_tools_local_ollama + runtime: nvidia + gpus: all + environment: + - NVIDIA_VISIBLE_DEVICES=all + - NVIDIA_DRIVER_CAPABILITIES=compute,utility + volumes: + - ollama_data:/root/.ollama + ports: + - "11434:11434" + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + mailhog: image: mailhog/mailhog:v1.0.0 container_name: scielo_tools_local_mailhog @@ -55,6 +76,7 @@ services: - redis - postgres - mailhog + - ollama ports: [] command: /start-celeryworker @@ -66,5 +88,9 @@ services: - redis - postgres - mailhog + - ollama ports: [] command: /start-celerybeat + +volumes: + ollama_data: diff --git a/pytest.ini b/pytest.ini index b8c192d..9963bee 100644 --- a/pytest.ini +++ b/pytest.ini @@ -3,3 +3,7 @@ DJANGO_SETTINGS_MODULE = config.settings.test python_files = tests.py test_*.py *_tests.py pythonpath = . addopts = --reuse-db +markers = + llama: integration tests that call local Llama/Ollama for reference marking +filterwarnings = + ignore:pkg_resources is deprecated as an API:UserWarning:packtools.catalogs diff --git a/reference/__init__.py b/reference/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/reference/__init__.py @@ -0,0 +1 @@ + diff --git a/reference/api/__init__.py b/reference/api/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/reference/api/__init__.py @@ -0,0 +1 @@ + diff --git a/reference/api/v1/__init__.py b/reference/api/v1/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/reference/api/v1/__init__.py @@ -0,0 +1 @@ + diff --git a/reference/api/v1/serializers.py b/reference/api/v1/serializers.py new file mode 100644 index 0000000..84dd184 --- /dev/null +++ b/reference/api/v1/serializers.py @@ -0,0 +1,50 @@ +from rest_framework import serializers + +OUTPUT_TYPE_CHOICES = ["json", "xml", "jats"] + + +class ReferencesInputField(serializers.Field): + def __init__(self, **kwargs): + kwargs.setdefault("style", {"base_template": "textarea.html", "rows": 10}) + super().__init__(**kwargs) + + def to_internal_value(self, data): + return data + + def to_representation(self, value): + return value + + +class ReferenceMarkRequestSerializer(serializers.Serializer): + references = ReferencesInputField( + help_text=( + "Uma referência por linha, lista JSON " + '["Ref A", "Ref B"] ou string única.' + ), + ) + type = serializers.ChoiceField( + choices=OUTPUT_TYPE_CHOICES, + default="json", + required=False, + help_text="Formato de saída: json, xml ou jats.", + ) + + +class ReferenceDocxRequestSerializer(serializers.Serializer): + file = serializers.FileField( + help_text="Arquivo .docx com secção de referências.", + ) + type = serializers.ChoiceField( + choices=OUTPUT_TYPE_CHOICES, + default="json", + required=False, + help_text="Formato de saída: json, xml ou jats.", + ) + + def validate_file(self, value): + name = (getattr(value, "name", "") or "").lower() + if not name.endswith(".docx"): + raise serializers.ValidationError("Only .docx files are accepted.") + if getattr(value, "size", None) == 0: + raise serializers.ValidationError("Empty file.") + return value diff --git a/reference/api/v1/views.py b/reference/api/v1/views.py new file mode 100644 index 0000000..84dcc3d --- /dev/null +++ b/reference/api/v1/views.py @@ -0,0 +1,113 @@ +from collections.abc import Mapping + +from django.http import JsonResponse +from rest_framework.decorators import action +from rest_framework.parsers import FormParser, MultiPartParser +from rest_framework.permissions import IsAuthenticated +from rest_framework.response import Response +from rest_framework.viewsets import GenericViewSet + +from reference.api.v1.serializers import ( + ReferenceDocxRequestSerializer, + ReferenceMarkRequestSerializer, +) +from reference.data_utils import build_ref_list, resolve_references_result +from reference.exceptions import ( + DocxReferencesError, + ReferenceLlamaDisabledError, + ReferenceLlamaMisconfiguredError, + ReferenceLlamaUnavailableError, +) +from reference.utils.references import parse_reference_list, references_from_docx_upload + + +class ReferenceViewSet(GenericViewSet): + serializer_class = ReferenceMarkRequestSerializer + permission_classes = [IsAuthenticated] + http_method_names = [ + "get", + "post", + "head", + "options", + ] + + def get_serializer_class(self): + if getattr(self, "action", None) == "docx": + return ReferenceDocxRequestSerializer + return ReferenceMarkRequestSerializer + + def create(self, request, *args, **kwargs): + return self.api_reference(request) + + def api_reference(self, request): + data = request.data + if not isinstance(data, Mapping): + return JsonResponse({"error": "Error processing"}, status=400) + + serializer = self.get_serializer(data=data) + if not serializer.is_valid(): + return JsonResponse(serializer.errors, status=400) + + post_references = serializer.validated_data.get("references") + post_type = serializer.validated_data.get("type", "json") + return self.mark_and_respond(post_references, post_type) + + @action( + detail=False, + methods=["get", "post"], + url_path="docx", + parser_classes=[MultiPartParser, FormParser], + ) + def docx(self, request): + if request.method == "GET": + return Response({}) + + serializer = self.get_serializer(data=request.data) + if not serializer.is_valid(): + return JsonResponse(serializer.errors, status=400) + + uploaded = serializer.validated_data["file"] + post_type = serializer.validated_data.get("type", "json") + try: + references = references_from_docx_upload(uploaded) + except DocxReferencesError as exc: + return JsonResponse({"error": str(exc)}, status=400) + return self.mark_and_respond(references, post_type) + + def mark_and_respond(self, references, output_type): + reference_list = parse_reference_list(references) + if not reference_list: + return JsonResponse({"error": "No references provided"}, status=400) + + try: + if output_type == "jats": + results = resolve_references_result( + references, + user=self.request.user, + output_type="xml", + ) + return JsonResponse({"ref_list": build_ref_list(results)}) + + results = resolve_references_result( + references, + user=self.request.user, + output_type=output_type, + ) + except ( + ReferenceLlamaDisabledError, + ReferenceLlamaMisconfiguredError, + ReferenceLlamaUnavailableError, + ) as exc: + return JsonResponse( + {"error": f"Llama model is not available: {exc}"}, + status=503, + ) + + if isinstance(references, str) and len(results) == 1: + response_data = { + "message": f"reference: {results[0]['data']}", + } + else: + response_data = {"references": results} + + return JsonResponse(response_data) diff --git a/reference/apps.py b/reference/apps.py new file mode 100644 index 0000000..41cdee6 --- /dev/null +++ b/reference/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class ReferenceConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "reference" diff --git a/reference/create_forms.py b/reference/create_forms.py new file mode 100644 index 0000000..c1e1564 --- /dev/null +++ b/reference/create_forms.py @@ -0,0 +1,47 @@ +from django import forms +from django.core.exceptions import ValidationError +from django.utils.translation import gettext_lazy as _ +from wagtail.admin.forms import WagtailAdminModelForm + +from reference.exceptions import DocxReferencesError +from reference.models import Reference +from reference.utils.references import references_from_docx_upload + + +class ReferenceCreateAdminForm(WagtailAdminModelForm): + docx_file = forms.FileField( + label=_("DOCX file"), + required=False, + help_text=_( + "Optional. If provided, references are extracted from the file " + "(takes precedence over the text field)." + ), + ) + + class Meta: + model = Reference + fields = ("mixed_citation",) + + def clean_docx_file(self): + docx_file = self.cleaned_data.get("docx_file") + if not docx_file: + return docx_file + name = (getattr(docx_file, "name", "") or "").lower() + if not name.endswith(".docx"): + raise ValidationError(_("Only .docx files are accepted.")) + if getattr(docx_file, "size", None) == 0: + raise ValidationError(_("Empty file.")) + return docx_file + + def clean(self): + cleaned = super().clean() + docx_file = cleaned.get("docx_file") + text = (cleaned.get("mixed_citation") or "").strip() + if docx_file: + try: + cleaned["mixed_citation"] = references_from_docx_upload(docx_file) + except DocxReferencesError as exc: + raise ValidationError(str(exc)) from exc + elif not text: + raise ValidationError(_("Provide reference text or a .docx file.")) + return cleaned diff --git a/reference/data_utils.py b/reference/data_utils.py new file mode 100644 index 0000000..3569a24 --- /dev/null +++ b/reference/data_utils.py @@ -0,0 +1,768 @@ +import hashlib +import json +import logging +import re + +from django.db import IntegrityError +from lxml import etree + +from reference.exceptions import ( + ReferenceLlamaDisabledError, + ReferenceLlamaMisconfiguredError, + ReferenceLlamaUnavailableError, +) +from reference.marking import mark_reference, mark_reference_texts, mark_references +from reference.models import ElementCitation, Reference, ReferenceStatus +from reference.utils.references import parse_reference_list, stz_norm + +logger = logging.getLogger(__name__) + +_UNSET = object() + + +def parse_marked_choice(choice): + if isinstance(choice, dict): + return choice + try: + return json.loads(choice) + except (TypeError, json.JSONDecodeError): + return {"raw": choice} + + +def is_non_reference(marked_data): + return isinstance(marked_data, dict) and marked_data.get("is_reference") is False + + +meses = { + "enero": "01", + "febrero": "02", + "marzo": "03", + "abril": "04", + "mayo": "05", + "junio": "06", + "julio": "07", + "agosto": "08", + "septiembre": "09", + "octubre": "10", + "noviembre": "11", + "diciembre": "12", + "january": "01", + "february": "02", + "march": "03", + "april": "04", + "may": "05", + "june": "06", + "july": "07", + "august": "08", + "september": "09", + "october": "10", + "november": "11", + "december": "12", + "jan": "01", + "feb": "02", + "mar": "03", + "apr": "04", + "jun": "06", + "jul": "07", + "aug": "08", + "sep": "09", + "oct": "10", + "nov": "11", + "dec": "12", + "janeiro": "01", + "fevereiro": "02", + "março": "03", + "abril": "04", + "maio": "05", + "junho": "06", + "julho": "07", + "agosto": "08", + "setembro": "09", + "outubro": "10", + "novembro": "11", + "dezembro": "12", +} + + +def get_number_of_month(texto): + texto = texto.lower() + for mes, numero in meses.items(): + if re.search(rf"\b{mes}\b", texto): + return numero + return None + + +def append_citation_pages(root, pages): + value = str(pages).strip().replace("–", "-").replace("—", "-") + if not value: + return + if "-" in value: + left, right = [part.strip() for part in value.split("-", 1)] + if left and right: + etree.SubElement(root, "fpage").text = left + etree.SubElement(root, "lpage").text = right + return + if value.lower().startswith("e") or (value.isdigit() and len(value) >= 5): + etree.SubElement(root, "elocation-id").text = value + return + etree.SubElement(root, "fpage").text = value + etree.SubElement(root, "lpage").text = value + + +def append_fpage_lpage(root, json_reference): + if "fpage" in json_reference: + etree.SubElement(root, "fpage").text = str(json_reference["fpage"]) + if "lpage" in json_reference: + etree.SubElement(root, "lpage").text = str(json_reference["lpage"]) + else: + etree.SubElement(root, "lpage").text = str(json_reference["fpage"]) + return True + if "pages" in json_reference: + append_citation_pages(root, json_reference["pages"]) + return True + return False + + +def normalize_doi(doi): + value = str(doi).strip() + for prefix in ( + "https://doi.org/", + "http://doi.org/", + "https://dx.doi.org/", + "http://dx.doi.org/", + "doi:", + ): + if value.lower().startswith(prefix): + value = value[len(prefix) :].strip() + break + return value.rstrip(".,;:)]}»\"'") + + +_DOI_URL_RE = re.compile( + r"https?://(?:dx\.)?doi\.org/(10\.\d{4,9}/\S+)", + re.IGNORECASE, +) +_DOI_LABEL_RE = re.compile( + r"\bdoi:\s*(10\.\d{4,9}/\S+)", + re.IGNORECASE, +) +_DOI_BARE_RE = re.compile( + r"(?\"')\]]+", re.IGNORECASE) + + +def normalize_uri(uri): + if uri is None: + return None + return str(uri).strip().rstrip(".,;:)]}»\"'") + + +def extract_uri_from_text(text, allow_doi=True): + if not text: + return None + for match in _URI_RE.finditer(str(text)): + uri = normalize_uri(match.group(0)) + if not uri: + continue + if not allow_doi and "doi.org/" in uri.lower(): + continue + return uri + return None + + +_VOL_ISSUE_PAGES_RE = re.compile( + r"(?= 4: + fields["fpage"] = match.group(3) + fields["lpage"] = match.group(4) + return fields + label = _ISSUE_LABEL_RE.search(value) + if label: + return {"num": int(label.group(1))} + return {} + + +def _missing(value): + return value in (None, "") + + +_WEB_LIKE = ("webpage", "software", "database", "legal-doc") + + +def enrich_marked_from_citation(marked_data, mixed_citation): + if not isinstance(marked_data, dict) or is_non_reference(marked_data): + return marked_data + marked = dict(marked_data) + reftype = marked.get("reftype") + + doi = marked.get("doi") + if doi not in (None, ""): + marked["doi"] = normalize_doi(str(doi)) + doi = marked["doi"] + else: + doi = None + if not doi: + found = extract_doi_from_text(mixed_citation) + if not found: + uri = marked.get("uri") + if uri and "doi.org/" in str(uri).lower(): + found = normalize_doi(str(uri)) + if found and reftype not in _WEB_LIKE: + marked["doi"] = found + doi = found + if doi: + uri = marked.get("uri") + if uri and "doi.org/" in str(uri).lower(): + marked.pop("uri", None) + + extracted = extract_vol_num_from_text(mixed_citation) + for key in ("vol", "num", "fpage", "lpage"): + if key in extracted and _missing(marked.get(key)): + marked[key] = extracted[key] + + if _missing(marked.get("uri")): + found_uri = extract_uri_from_text( + mixed_citation, + allow_doi=reftype in _WEB_LIKE, + ) + if found_uri: + if "doi.org/" in found_uri.lower() and reftype not in _WEB_LIKE: + if _missing(marked.get("doi")): + marked["doi"] = normalize_doi(found_uri) + elif "doi.org/" in found_uri.lower() and not _missing(marked.get("doi")): + pass + else: + marked["uri"] = found_uri + elif marked.get("uri"): + marked["uri"] = normalize_uri(marked["uri"]) + + return marked + + +def marked_gained_fields(before, after): + if not isinstance(before, dict) or not isinstance(after, dict): + return False + for key in ("doi", "num", "vol", "fpage", "lpage", "uri"): + if not _missing(after.get(key)) and _missing(before.get(key)): + return True + if before.get("uri") and "uri" not in after and after.get("doi"): + return True + return False + + +def append_doi(root, doi): + value = normalize_doi(doi) + if not value: + return + etree.SubElement(root, "pub-id", attrib={"pub-id-type": "doi"}).text = value + + +def append_person_group(root, people, person_group_type): + person_group = etree.SubElement( + root, + "person-group", + attrib={"person-group-type": person_group_type}, + ) + for person in people: + if "collab" in person and "surname" not in person and "fname" not in person: + etree.SubElement(person_group, "collab").text = person["collab"] + continue + name = etree.Element("name") + if "surname" in person: + etree.SubElement(name, "surname").text = person["surname"] + if "fname" in person: + etree.SubElement(name, "given-names").text = person["fname"] + if "collab" in person: + etree.SubElement(name, "collab").text = person["collab"] + person_group.append(name) + + +def append_num_pages(root, num_pages): + value = str(num_pages).strip() + if not value: + return + digits = re.search(r"\d+", value) + size = etree.SubElement(root, "size", attrib={"units": "pages"}) + size.text = digits.group() if digits else value + + +def append_ext_link(root, uri): + etree.SubElement( + root, + "ext-link", + attrib={ + "ext-link-type": "uri", + "{http://www.w3.org/1999/xlink}href": uri, + }, + ).text = uri + + +def append_access_date(root, access_date): + match = re.search(r"\b\d{4}\b", access_date) + if not match: + etree.SubElement( + root, + "date-in-citation", + attrib={"content-type": "access-date"}, + ).text = access_date + return + year = match.group() + month = get_number_of_month(access_date) or "01" + etree.SubElement( + root, + "date-in-citation", + attrib={ + "content-type": "access-date", + "iso-8601-date": year + "-" + month + "-00", + }, + ).text = access_date + + +def get_xml(json_reference): + try: + json_reference = json.loads(json_reference) + except json.JSONDecodeError as exc: + logger.error("Malformed JSON from IA: %s", exc) + return etree.Element("error") + + reftype = json_reference.get("reftype") + if not reftype: + logger.error("Missing reftype in IA JSON: %s", json_reference) + return etree.Element("error") + + needs_xlink = reftype in ( + "webpage", + "data", + "software", + "database", + ) or bool(json_reference.get("uri")) + if needs_xlink: + root = etree.Element( + "element-citation", + attrib={"publication-type": reftype}, + nsmap={"xlink": "http://www.w3.org/1999/xlink"}, + ) + else: + root = etree.Element( + "element-citation", + attrib={"publication-type": reftype}, + ) + + if "authors" in json_reference: + append_person_group(root, json_reference["authors"], "author") + if "editors" in json_reference: + append_person_group(root, json_reference["editors"], "editor") + + if reftype == "journal": + if "title" in json_reference: + etree.SubElement(root, "article-title").text = json_reference["title"] + if "source" in json_reference: + etree.SubElement(root, "source").text = json_reference["source"] + if "vol" in json_reference: + etree.SubElement(root, "volume").text = str(json_reference["vol"]) + if "num" in json_reference: + etree.SubElement(root, "issue").text = str(json_reference["num"]) + append_fpage_lpage(root, json_reference) + if "doi" in json_reference: + append_doi(root, json_reference["doi"]) + + if reftype == "book": + chapter = json_reference.get("chapter") or json_reference.get("chapter_title") + if chapter: + etree.SubElement(root, "part-title").text = chapter + if "source" in json_reference: + etree.SubElement(root, "source").text = json_reference["source"] + elif "title" in json_reference and not chapter: + etree.SubElement(root, "source").text = json_reference["title"] + if "edition" in json_reference: + etree.SubElement(root, "edition").text = str(json_reference["edition"]) + if "vol" in json_reference: + etree.SubElement(root, "volume").text = str(json_reference["vol"]) + append_fpage_lpage(root, json_reference) + if "organization" in json_reference: + etree.SubElement(root, "publisher-name").text = json_reference[ + "organization" + ] + elif "publisher" in json_reference: + etree.SubElement(root, "publisher-name").text = json_reference["publisher"] + publisher_loc = json_reference.get("location") or json_reference.get( + "org_location" + ) + if publisher_loc: + etree.SubElement(root, "publisher-loc").text = publisher_loc + if "num_pages" in json_reference: + append_num_pages(root, json_reference["num_pages"]) + if "doi" in json_reference: + append_doi(root, json_reference["doi"]) + + if reftype == "thesis": + if "source" in json_reference: + etree.SubElement(root, "source").text = json_reference["source"] + elif "title" in json_reference: + etree.SubElement(root, "source").text = json_reference["title"] + if "degree" in json_reference: + etree.SubElement( + root, "comment", attrib={"content-type": "degree"} + ).text = json_reference["degree"] + if "organization" in json_reference: + etree.SubElement(root, "publisher-name").text = json_reference[ + "organization" + ] + if "location" in json_reference: + etree.SubElement(root, "publisher-loc").text = json_reference["location"] + if "num_pages" in json_reference: + append_num_pages(root, json_reference["num_pages"]) + + if reftype == "confproc": + if "title" in json_reference: + etree.SubElement(root, "conf-name").text = json_reference["title"] + elif "conf_name" in json_reference: + etree.SubElement(root, "conf-name").text = json_reference["conf_name"] + if "source" in json_reference: + etree.SubElement(root, "source").text = json_reference["source"] + conf_loc = json_reference.get("conf_loc") or json_reference.get("location") + if conf_loc: + etree.SubElement(root, "conf-loc").text = conf_loc + if "conf_date" in json_reference: + etree.SubElement(root, "conf-date").text = str(json_reference["conf_date"]) + conf_num = json_reference.get("conf_num") + if conf_num is None and "num" in json_reference: + conf_num = json_reference["num"] + if conf_num is not None: + etree.SubElement(root, "conf-num").text = str(conf_num) + if "organization" in json_reference: + etree.SubElement(root, "publisher-name").text = json_reference[ + "organization" + ] + if "org_location" in json_reference: + etree.SubElement(root, "publisher-loc").text = json_reference[ + "org_location" + ] + if "num_pages" in json_reference: + append_num_pages(root, json_reference["num_pages"]) + elif "pages" in json_reference and "fpage" not in json_reference: + append_num_pages(root, json_reference["pages"]) + if "doi" in json_reference: + append_doi(root, json_reference["doi"]) + + if reftype == "data": + if "title" in json_reference: + etree.SubElement(root, "data-title").text = json_reference["title"] + if "source" in json_reference: + etree.SubElement(root, "source").text = json_reference["source"] + if "version" in json_reference: + etree.SubElement(root, "version").text = str(json_reference["version"]) + if "uri" in json_reference: + append_ext_link(root, json_reference["uri"]) + if "organization" in json_reference: + etree.SubElement(root, "publisher-name").text = json_reference[ + "organization" + ] + if "location" in json_reference: + etree.SubElement(root, "publisher-loc").text = json_reference["location"] + if "access_id" in json_reference: + etree.SubElement(root, "comment").text = str(json_reference["access_id"]) + if "doi" in json_reference: + append_doi(root, json_reference["doi"]) + if "access_date" in json_reference: + append_access_date(root, json_reference["access_date"]) + + if reftype in ("webpage", "software", "database", "legal-doc"): + if "title" in json_reference: + etree.SubElement(root, "source").text = json_reference["title"] + elif "source" in json_reference and root.find("source") is None: + etree.SubElement(root, "source").text = json_reference["source"] + if "uri" in json_reference: + append_ext_link(root, json_reference["uri"]) + if "organization" in json_reference: + etree.SubElement(root, "publisher-name").text = json_reference[ + "organization" + ] + publisher_loc = ( + json_reference.get("location") + or json_reference.get("country") + or json_reference.get("org_location") + ) + if publisher_loc: + etree.SubElement(root, "publisher-loc").text = publisher_loc + if "version" in json_reference and reftype == "software": + etree.SubElement(root, "version").text = str(json_reference["version"]) + if "access_id" in json_reference: + etree.SubElement(root, "comment").text = str(json_reference["access_id"]) + if "doi" in json_reference: + append_doi(root, json_reference["doi"]) + if "access_date" in json_reference: + append_access_date(root, json_reference["access_date"]) + + if reftype == "other": + if "title" in json_reference: + etree.SubElement(root, "source").text = json_reference["title"] + elif "source" in json_reference: + etree.SubElement(root, "source").text = json_reference["source"] + if "doi" in json_reference: + append_doi(root, json_reference["doi"]) + if "uri" in json_reference: + append_ext_link(root, json_reference["uri"]) + + if "date" in json_reference: + etree.SubElement(root, "year").text = str(json_reference["date"]) + + if json_reference.get("doi") and root.find("pub-id[@pub-id-type='doi']") is None: + append_doi(root, json_reference["doi"]) + + if ( + json_reference.get("uri") + and root.find("ext-link") is None + and root.find("pub-id[@pub-id-type='doi']") is None + ): + append_ext_link(root, json_reference["uri"]) + + return root + + +def build_ref_list(results): + root = etree.Element("ref-list") + etree.SubElement(root, "title").text = "References" + + for index, item in enumerate(results, start=1): + ref = etree.SubElement(root, "ref", attrib={"id": f"B{index}"}) + mixed = etree.SubElement(ref, "mixed-citation") + mixed.text = item.get("mixed_citation") or "" + + marked_xml = item.get("data") or "" + if not marked_xml: + logger.warning("Missing element-citation XML for ref B%s", index) + continue + try: + citation_node = etree.fromstring(marked_xml.encode("utf-8")) + except etree.XMLSyntaxError as exc: + logger.warning("Invalid element-citation XML for ref B%s: %s", index, exc) + continue + if citation_node.tag == "error": + logger.warning("Error element-citation for ref B%s", index) + continue + ref.append(citation_node) + + return etree.tostring(root, pretty_print=True, encoding="unicode") + + +def resolve_reference_result( + mixed_citation, user=None, output_type="json", marked_data=_UNSET +): + normalized = stz_norm(mixed_citation) + checksum = hashlib.sha256(normalized.encode("utf-8")).hexdigest() + + try: + reference = Reference.objects.get(checksum=checksum) + except Reference.DoesNotExist: + if marked_data is _UNSET: + marked_data = None + for choice in mark_reference(mixed_citation): + marked_data = parse_marked_choice(choice) + break + if marked_data is None or is_non_reference(marked_data): + logger.info("Ignoring non-reference input: %r", mixed_citation) + return None + marked_data = enrich_marked_from_citation(marked_data, mixed_citation) + if not marked_data.get("reftype"): + logger.info( + "Ignoring mark without reftype: %r marked=%s", + mixed_citation, + marked_data, + ) + return None + try: + reference, created = Reference.objects.get_or_create( + checksum=checksum, + defaults={ + "mixed_citation": mixed_citation, + "status": ReferenceStatus.CREATING, + "creator": user, + }, + ) + except IntegrityError: + reference = Reference.objects.get(checksum=checksum) + created = False + if created or not reference.element_citation.exists(): + ElementCitation.objects.create( + reference=reference, + marked=marked_data, + marked_xml=etree.tostring( + get_xml(json.dumps(marked_data)), + pretty_print=True, + encoding="unicode", + ), + ) + reference.status = ReferenceStatus.READY + reference.save() + + element = reference.element_citation.first() + marked = element.marked if element else {} + if isinstance(marked, dict): + enriched = enrich_marked_from_citation(marked, reference.mixed_citation) + if marked_gained_fields(marked, enriched): + marked_xml = etree.tostring( + get_xml(json.dumps(enriched)), + pretty_print=True, + encoding="unicode", + ) + element.marked = enriched + element.marked_xml = marked_xml + element.save(update_fields=["marked", "marked_xml"]) + marked = enriched + if output_type in ("xml", "jats"): + data = element.marked_xml if element else "" + else: + data = marked if element else {} + + return { + "mixed_citation": reference.mixed_citation, + "data": data, + } + + +def resolve_references_result(references, user=None, output_type="json"): + citations = parse_reference_list(references) + pending = [] + pending_seen = set() + for citation in citations: + checksum = hashlib.sha256(stz_norm(citation).encode("utf-8")).hexdigest() + if Reference.objects.filter(checksum=checksum).exists(): + continue + if citation in pending_seen: + continue + pending_seen.add(citation) + pending.append(citation) + + premarked = {} + if pending: + for citation, content in zip(pending, mark_reference_texts(pending)): + premarked[citation] = ( + parse_marked_choice(content) if content is not None else None + ) + + results = [] + for citation in citations: + if citation in premarked: + item = resolve_reference_result( + citation, + user=user, + output_type=output_type, + marked_data=premarked[citation], + ) + else: + item = resolve_reference_result( + citation, user=user, output_type=output_type + ) + if item is not None: + results.append(item) + return results + + +def get_reference(obj_id): + logger.info("Starting get_reference for ID=%s", obj_id) + obj_reference = Reference.objects.get(id=obj_id) + try: + logger.info("Marking citation: %r", obj_reference.mixed_citation) + marked = list(mark_references(obj_reference.mixed_citation)) + + citations_created = 0 + for item in marked: + for i in item["choices"]: + marked_data = enrich_marked_from_citation( + parse_marked_choice(i), + item.get("references") or obj_reference.mixed_citation, + ) + if is_non_reference(marked_data): + logger.info( + "Skipping non-reference mark for ID=%s: %r", + obj_id, + item.get("references"), + ) + continue + if not marked_data.get("reftype") and "raw" not in marked_data: + logger.info( + "Skipping mark without reftype for ID=%s: %s", + obj_id, + marked_data, + ) + continue + citation = ElementCitation.objects.create( + reference=obj_reference, + marked=marked_data, + marked_xml=etree.tostring( + get_xml(json.dumps(marked_data)), + pretty_print=True, + encoding="unicode", + ), + ) + citations_created += 1 + logger.debug( + "Created ElementCitation ID=%s for marked citation: %s", + citation.pk, + i, + ) + + if citations_created == 0: + logger.info( + "No bibliographic citations for ID=%s; deleting Reference", + obj_id, + ) + obj_reference.delete() + return + + obj_reference.status = ReferenceStatus.READY + obj_reference.save() + logger.info( + "get_reference completed for ID=%s. Citations created: %d", + obj_id, + citations_created, + ) + except ( + ReferenceLlamaDisabledError, + ReferenceLlamaMisconfiguredError, + ReferenceLlamaUnavailableError, + ) as exc: + logger.error( + "Llama unavailable in get_reference for ID=%s: %s — deleting Reference", + obj_id, + exc, + ) + obj_reference.delete() + raise + except Exception as exc: + logger.error("Error in get_reference for ID=%s: %s", obj_id, exc, exc_info=True) + raise diff --git a/reference/exceptions.py b/reference/exceptions.py new file mode 100644 index 0000000..cb3de61 --- /dev/null +++ b/reference/exceptions.py @@ -0,0 +1,14 @@ +class ReferenceLlamaDisabledError(Exception): + pass + + +class ReferenceLlamaMisconfiguredError(Exception): + pass + + +class ReferenceLlamaUnavailableError(Exception): + pass + + +class DocxReferencesError(Exception): + pass diff --git a/reference/fixtures/__init__.py b/reference/fixtures/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/reference/fixtures/__init__.py @@ -0,0 +1 @@ + diff --git a/reference/fixtures/references.py b/reference/fixtures/references.py new file mode 100644 index 0000000..a01f063 --- /dev/null +++ b/reference/fixtures/references.py @@ -0,0 +1,10 @@ +from pathlib import Path + +from reference.utils.references import parse_reference_list + +_FIXTURES_DIR = Path(__file__).resolve().parent + +REFERENCES = parse_reference_list( + (_FIXTURES_DIR / "references.txt").read_text(encoding="utf-8") +) +REF_LIST_XML = (_FIXTURES_DIR / "references.xml").read_text(encoding="utf-8") diff --git a/reference/fixtures/references.txt b/reference/fixtures/references.txt new file mode 100644 index 0000000..1671eb3 --- /dev/null +++ b/reference/fixtures/references.txt @@ -0,0 +1,103 @@ +2.Hou, Y., Chen, L., Li, Z., Zhao, G. and Zhang, C., (2020). Effects of artificial aging on microstructure, mechanical properties and stress corrosion cracking of a novel high strength 7A99 Al alloy. Materials Science and Engineering: A, 780, 139217. +23.He, X., Pan, Q., Li, H., Huang, Z., Liu, S., Li, K. and Li, X., (2019). Effect of artificial aging, delayed aging, and pre-aging on microstructure and properties of 6082 aluminum alloy. Metals, 9(2), 173. +24.Mukund, A., Nair, A.S., Nived, S., Raagavendran, R., Premkumar, A., Raj, A.N. and Shankar, K.V., (2021). Impact of solutionising temperature on the microstructure, hardness and tensile strength of Al-6.6 Si-0.3 Mg-3Ni alloys. Materials Today: Proceedings, 38, 2117-2122. +Ab’Saber, A. N. (2003). Os domínios da natureza no Brasil: Potencialidades paisagísticas. Ateliê Editorial. +Alvares, C. A., Stape, J. L., Sentelhas, P. C., & Gonçalves, J. L. M. (2013a). Modeling monthly mean air temperature for Brazil. Theoretical and Applied Climatology, 113, 407–427. https://doi.org/10.1007/s00704-012-0796-6 +Alvares, C. A., Stape, J. L., Sentelhas, P. C., Gonçalves, J. L. M., & Sparovek, G. (2013b). Köppen’s climate classification map for Brazil. Meteorologische Zeitschrift, 22(6), 711–728. https://doi.org/10.1127/0941-2948/2013/0507 +Anderson, J. M., & Ingram, J. S. I. (1993). Tropical soil biology and fertility: A handbook of methods (2nd ed.). CAB International. +Anderson, A. B., & Pires, J. M. (1978). Study of the buriti palm (Mauritia flexuosa) in the Amazon and its ecological significance. Acta Amazonica, 8(4), 493–502. +Andrade, T. M., Assis, R. L., Wittmann, F., Schöngart, J., & Piedade, M. T. F. (2008). Padrões de regeneração em clareiras de origem antrópica na várzea da RDS Mamirauá, Amazônia Central. Uakari, 4(2), 19–32. +Assis, R. L., Wittmann, F., Piedade, M. T. F., … (2015). Effects of hydroperiod and substrate properties on tree alpha diversity and composition in Amazonian floodplain forests. Plant Ecology, 216, 41–54. https://doi.org/10.1007/s11258-014-0415-y +Assis, R. L., Wittmann, F., Bredin, Y. K., … (2019). Aboveground woody biomass distribution in Amazonian floodplain forests: Effects of hydroperiod and substrate properties. Forest Ecology and Management, 432, 365–375. https://doi.org/10.1016/j.foreco.2018.09.031 +Augie, B. (2017). gridExtra: Miscellaneous functions for "Grid" graphics (Version 2.3) [R package]. https://CRAN.R-project.org/package=gridExtra +Ayres, J. M. C. (1995). As matas de várzea do Mamirauá. Sociedade Civil Mamirauá. +Brasil. (2024). Decreto nº 91.886, de 05 de novembro de 1985. Diário Oficial da União. https://www.planalto.gov.br/cCivil_03/Atos/decretos/1985/D91886 +Batista, E. S., & Cunha, C. N. (2012). Structure and floristic composition of Mauritia flexuosa palm swamps (buritizais) in the Pantanal, Brazil. Acta Botanica Brasilica, 26(3), 539–552. +Brightsmith, D. J., & Bravo, A. (2006). Ecology and management of nesting blueandyellow macaws (Ara ararauna) in Mauritia palm swamps. Biodiversity and Conservation, 15(12), 4271–4287. https://doi.org/10.1007/s10531-005-3579-x +Brito, J. M., Wittmann, F., Schöngart, J., … (2008). Guia de 42 espécies madeireiras da Reserva de Desenvolvimento Sustentável Mamirauá. Sociedade Civil Mamirauá. +Brower, J. E., & Zar, J. H. (1984). Field and laboratory methods for general ecology (3rd ed.). WMC Brown Publishers. +Casanova, M. T., & Brock, M. A. (2000). How do depth, duration and frequency of flooding influence the establishment of wetland plant communities? Plant Ecology, 147(2), 237–250. https://doi.org/10.1023/A:1009875226637 +Cardoso, D. T., Särkinen, T., Alexander, S., … (2017). Amazon plant diversity revealed by a taxonomically verified species list. Proceedings of the National Academy of Sciences, 114(40), 10695–10700. https://doi.org/10.1073/pnas.1706756114 +Connell, J. H., & Lowman, M. D. (1989). Lowdensity tropical rain forests: Some possible mechanisms for their existence. The American Naturalist, 134, 88–119. +Correia, A. H., & Diretoria de Serviço Geográfico do Exército. (2011). Metodologias e resultados preliminares do Projeto Radiografia da Amazônia. Anais do XV Simpósio Brasileiro de Sensoriamento Remoto. +Dambros, C. S. (2020). csdambros/Rfunctions: First release (Version 1.0) [Computer software]. Zenodo. https://doi.org/10.5281/zenodo.3784397 +Dinno, A. (2017). dunn.test: Dunn's test of multiple comparisons using rank sums (Version 1.3.5) [R package]. https://CRAN.R-project.org/package=dunn.test +Durgante, F. M., Higuchi, N., Ohashi, S., … (2023). Soil fertility and drought interact to determine large variations in wood production for a hyperdominant Amazonian tree species. Frontiers in Forests and Global Change, 5, Article 1065645. https://doi.org/10.3389/ffgc.2022.1065645 +Emilio, T., Nelson, B. W., & Prado, P. I. (2013). Unexpected monoculture in a hyperdiverse tropical forest. Scientific Reports, 3, 3267. https://doi.org/10.1038/srep03267 +Engelbrecht, B. M., VásquezRivera, T., Tyree, M. T., Kursar, T. A., & ChambiSalazar, A. (2007). Drought effects on seedling survival in a tropical moist forest. Trees, 21(5), 389–398. https://doi.org/10.1007/s00468-004-0393-0 +Felix Ribeiro, K. A. (2025). Replication data for: Mauritia flexuosa (Buriti) forest description: Hydro-edaphic effects on tree species distribution in Western Amazon. SciELO Data. Draft version. https://doi.org/10.48331/SCIELODATA.RIVAW4 +Ferreira, L. V., & Stohlgren, T. J. (1999). Effects of river level fluctuation on plant species richness, diversity, and distribution in a floodplain forest in Central Amazonia. Oecologia, 120(4), 582–587. +Fleischmann, A. S., Alves, P. C., & Nascimento, A. Z. (2023). Caracterização geográfica da Área de Relevante Interesse Ecológico JavariBuriti. In Instituto de Desenvolvimento Sustentável Mamirauá (Ed.), Projeto JavariBuriti: Subsídios para a elaboração do Plano de Manejo da Área de Relevante Interesse Ecológica JavariBuriti (pp. 15–48). Instituto de Desenvolvimento Sustentável Mamirauá. +Flora e Funga do Brasil. (2022). Jardim Botânico do Rio de Janeiro. Retrieved December 19, 2022, from http://floradobrasil.jbrj.gov.br +Galeano, A., Urrego, L. E., Sánchez, M., … (2015). Environmental drivers for regeneration of Mauritia flexuosa Lf in Colombian Amazonian swamp forest. Aquatic Botany, 123, 47–53. https://doi.org/10.1016/j.aquabot.2015.02.001 +Graham, J. (2003). HH: Regression and other stories [R package]. https://CRAN.R-project.org/package=HH +Gotelli, N. J., & Colwell, R. K. (2001). Quantifying biodiversity: Procedures and pitfalls in the measurement and comparison of species richness. Ecology Letters, 4, 379–391. https://doi.org/10.1046/j.1461-0248.2001.00230.x +Hamilton, S. K., Sippel, S. J., & Melack, J. M. (2007). Seasonal inundation patterns in two large savanna floodplains of South America: The Llanos de Moxos (Bolivia) and the Llanos del Orinoco (Venezuela and Colombia). Hydrological Processes, 21(14), 1719–1736. https://doi.org/10.1002/hyp.5559 +Hart, T. B., Hart, J. A., & Murphy, P. G. (1989). Monodominant and speciesrich forests of the humid tropics: Causes for their cooccurrence. The American Naturalist, 133(5), 613–633. +Hess, L. L. (2003). Mapping wetlands in the Amazon Basin using SAR data. In Proceedings of the IEEE International Geoscience and Remote Sensing Symposium (IGARSS ’03) (Vol. 2, pp. 1375–1377). IEEE. +Householder, J. E., Schumann, D. A., & Hamilton, S. K. (2010). Influence of hydrologic connectivity and environmental heterogeneity on floodplain wetland vegetation patterns in the Amazon River basin. Wetlands, 30(6), 1167–1180. +Householder, J. E., Schumann, D. A., & Hamilton, S. K. (2012). Assessing inundation impacts in floodplain wetlands: Relations between flood duration and plant community composition. Wetlands, 32(6), 1091–1103. +Instituto Brasileiro de Geografia e Estatística (IBGE). (2019a). Índice de Organização do Território / Malhas Territoriais / Malhas Municipais / Município 2015 / Brasil [Data set]. Retrieved August 28, 2019, from ftp://geoftp.ibge.gov.br/organizacao_do_territorio/malhas_territoriais/malhas_municipais/municipio_2015/Brasil/BR/ +Instituto Brasileiro de Geografia e Estatística (IBGE). (2019b). Bases cartográficas contínuas BC250 (versão 2017) – Shapefiles [Data set]. Retrieved August 28, 2019, from http://geoftp.ibge.gov.br/cartas_e_mapas/bases_cartograficas_continuas/bc250/versao2017/shapefile/ +Junk, W. J. (1997). General aspects of floodplain ecology with special reference to Amazonian floodplains. In W. J. Junk (Ed.), The Central Amazon floodplain: Ecology of a pulsing system (Ecological Studies 126, pp. 3–20). Springer. +Junk, W. J., Bayley, P. B., & Sparks, R. E. (1989). The flood pulse concept in river–floodplain systems. Canadian Special Publication of Fisheries and Aquatic Sciences, 106(1), 110–127. +Junk, W. J., & Furch, K. (1993). The physical and chemical limnology of Amazonian waters and their role in the carbon cycle. In G. T. Prance & T. E. Lovejoy (Eds.), Amazonia (pp. 385–404). Pergamon Press. +Junk, W. J., Piedade, M. T. F., Schöngart, J., CohnHaft, M., Adeney, J. M., & Wittmann, F. (2011). A classification of major naturally occurring Amazonian lowland wetlands. Wetlands, 31(4), 623–640. https://doi.org/10.1007/s13157-011-0190-7 +Junk, W. J., Piedade, M. T. F., Lourival, R., Wittmann, F., Kandus, P., Lacerda, L. D., Bozelli, R. L., Esteves, F. A., Nunes da Cunha, C., Maltchik, L., Schöngart, J., Schaeffer-Novelli, Y., & Agostinho, A. A. (2014). Brazilian wetlands: Their definition, delineation, and classification for research, sustainable management, and protection. Aquatic Conservation: Marine and Freshwater Ecosystems, 24(1), 5–22. https://doi.org/10.1002/aqc.2386 +Kahn, F., & Mejia, K. (1990). Palm communities in wetland forest ecosystems of Peruvian Amazonia. Forest Ecology and Management, 33–34, 169–179. https://doi.org/10.1016/0378-1127(90)90195-X +Kahn, F., & de Granville, J.-J. (1992). Palms in forest ecosystems of Amazonia. Springer-Verlag. +Kelly, T. J., Baird, A. J., Roucoux, K. H., Baker, T. R., HonorioCoronado, E. M., Ríos, M., & Lawson, I. T. (2014). The high hydraulic conductivity of three wooded tropical peat swamps in northeast Peru: Measurements and implications for hydrological function. Hydrological Processes, 28, 3373–3387. https://doi.org/10.1002/hyp.9884 +Klimas, C. A., Kainer, K. A., Oliveira, W., & Lúcia, H. (2012). The economic value of sustainable seed and timber harvests of multiuse species: An example using Carapa guianensis. Forest Ecology and Management, 268, 81–91. https://doi.org/10.1016/j.foreco.2011.03.006 +Kubitzki, K. (1989). The ecology of Amazonian palm swamps. In K. Kubitzki (Ed.), Tropical rain forest ecosystems: Biogeographical and ecological studies (pp. 227–239). Springer. +Lähteenoja, O., Ruokolainen, K., Schulman, L., … (2012). Amazonian peatlands: An ignored C sink and potential source. Global Change Biology, 18(12), 3366–3376. https://doi.org/10.1111/j.1365-2486.2009.01920.x +Lorenzi, H. (2002). Árvores brasileiras: Manual de identificação e cultivo de plantas arbóreas nativas do Brasil (Vol. 1). Editora Plantarum. +Magurran, A. E. (1988). Ecological diversity and its measurement. Princeton University Press. +Magnussen, S., Kleinn, C., & Picard, N. (2001). A review of the concept of cluster sampling in forest inventories. Canadian Journal of Forest Research, 31(7), 1105–1115. https://doi.org/10.1139/x01-054 +Marshall, B. G., Forsberg, B. R., Hess, L. L., & CarvalhoFreitas, C. E. (2011). Water temperature differences in interfluvial palm swamp habitats of Paracheirodon axelrodi and P. simulans (Osteichthyes: Characidae) in the middle Rio Negro, Brazil. Ichthyological Exploration of Freshwaters, 22(4), 377–383. +Mendiburu, F. (2023). agricolae: Statistical procedures for agricultural research (Version 1.37) [R package]. https://CRAN.R-project.org/package=agricolae +Milborrow, S. (2018). rpart.plot: Plot “rpart” models—An enhanced version of plot.rpart [R package]. Retrieved June 30, 2023, from https://CRAN.R-project.org/package=rpart.plot +Moreira, D. M. (2016). Geodésia aplicada ao monitoramento hidrológico da bacia Amazônica [Doctoral dissertation, Universidade Federal do Rio de Janeiro]. +MuellerDombois, D., & Ellenberg, H. (1974). Aims and methods of vegetation ecology. John Wiley & Sons. +Oliveira, R. L. C., Ricardo, P., Veridiana, S., & Barbosa, R. I. (2017). Structure and tree species composition in different habitats of savanna used by indigenous people in the Northern Brazilian Amazon. Biodiversity Data Journal, 5, Article e20044. https://doi.org/10.3897/BDJ.5.e20044 +Oliveira, R. L. C., Scudeller, V. V., Barbosa, R. I., … (2019). Aparência ecológica e conservação de espécies lenhosas pelos Makuxis nas savanas de Roraima, Amazônia brasileira. Ethnoscientia – Brazilian Journal of Ethnobiology and Ethnoecology, 4, 1–14. http://dx.doi.org/10.18542/ethnoscientia.v0i0.10245 +Oksanen, J., Blanchet, F. G., Friendly, M., … (2018). vegan: Community ecology package (Version 2.53) [R package]. Retrieved June 30, 2023, from https://CRAN.R-project.org/package=vegan +Oksanen, J., Blanchet, F. G., Friendly, M., … (2022). vegan: Community ecology package (Version 2.64) [R package]. Retrieved June 30, 2023, from https://cran.r-project.org/package=vegan +Oliveira, N. A., & Amaral, I. L. (2005). Aspectos florísticos, fitossociológicos e ecológicos de um subbosque de terra firme na Amazônia Central, Amazonas, Brasil. Acta Amazônica, 35(1), 1–16. https://doi.org/10.1590/S0044-59672005000100002 +Oyelade, J. O., Idowu, D. O., Oniya, O. O., & Ogunkunle, D. O. (2017). Optimization of biodiesel production from sandbox (Hura crepitans L.) seed oil using two different catalysts. Energy Sources, Part A: Recovery, Utilization, and Environmental Effects, 39(12), 1242–1249. https://doi.org/10.1080/15567036.2017.1320691 +Pella, E. (1990). Elemental organic analysis. Part 2. State of the art. American Laboratory, 22, 28–32. +Perkins, J. L. (1982). ShannonWeaver or ShannonWiener. Journal of the Water Pollution Control Federation, 54(7), 1049–1050. +Pereira, M. G., Valladares, G. S., Anjos, L. H. C., Benites, V. M., EspíndulaJúnior, A., & Ebeling, A. G. (2006). Organic carbon determination in Histosols and soil horizons with high organic matter content from Brazil. Scientia Agricola, 63(2), 187–193. https://doi.org/10.1590/S0103-90162006000200012 +Pintaud, J.-C., Dransfield, J., Henderson, A., Borchsenius, F., Mogens, M. B., Salo, J., & Balslev, H. (2008). A revision of the palm genera (Arecaceae). Botanical Journal of the Linnean Society, 157(1), 1–68. https://doi.org/10.1111/j.1095-8339.2008.00728.x +Poggio, L., De Sousa, L. M., Batjes, N. H., Heuvelink, G. B. M., Kempen, B., Ribeiro, E., & Rossiter, D. (2021). SoilGrids 2.0: Producing soil information for the globe with quantified spatial uncertainty. Soil, 7, 217–240. +Quesada, C. A., Phillips, O. L., Schwarz, M., … (2012). Basinwide variations in Amazon forest structure and function are mediated by both soils and climate. Biogeosciences, 9(6), 2203–2246. https://doi.org/10.5194/bg-9-2203-2012 +R Core Team. (2021). R: A language and environment for statistical computing [Computer software]. R Foundation for Statistical Computing. Retrieved May 14, 2023, from https://www.R-project.org/ +Resende, I. L. M., Santos, F. P., Chaves, L. J., & Nascimento, J. L. (2012). Estrutura etária de populações de Mauritia flexuosa L. (Arecaceae) de veredas da região central de Goiás, Brasil. Revista Árvore, 36(1), 103–112. https://doi.org/10.1590/S0100-67622012000100012 +Rigueira, S., Brina, A. E., Filho, J. R., CostaSilva, L. V., Bedê, L. C., & Rezende, M. (2002). Projeto Buriti: Artesanato, natureza e sociedade. Instituto Terra Brasilis de Desenvolvimento SócioAmbiental. +Rosa, R., Barbosa, R. I., & Koptur, S. (2014). Which factors explain reproductive output of Mauritia flexuosa (Arecaceae) in forest and savanna habitats of northern Amazonia? International Journal of Plant Sciences, 175(3), 307–318. https://doi.org/10.1086/674446 +Salinas, H., & RamírezDelgado, D. (2021). ecolTest: Community ecology tests [R package]. +Sander, N. L. (2014). Estrutura, composição florística e etnobiologia de um buritizal na fronteira biológica AmazôniaCerrado [Master’s thesis, Universidade do Estado de Mato Grosso]. +Sander, N. L., Ribeiro, R. S., Silva, D. R., … (2017). Floristic, phytosociology and spatial distribution of a monodominant Mauritia flexuosa L. f. forest in a Southern Amazon in the Arc of deforestation. In Natural resources in wetlands: From Pantanal to Amazonia (pp. – ). MPEG. +Sanquetta, C. R., Corte, A. P. D., & Netto, S. P. (2014). Inventários florestais: Planejamento e execução. Universidade Federal do Paraná (UFPR). +Schaap, K. J., Fuchslueger, L., Hoosbeek, M. R., Hofhansl, F., Martins, N. P., ValverdeBarrantes, O. J., & Quesada, C. A. (2021). Litter inputs and phosphatase activity affect the temporal variability of organic phosphorus in a tropical forest soil in the Central Amazon. Plant and Soil, 469(1–2), 423–441. +Silva, V. P., CarvalhoBrito, L., Marques, A. M., CunhaCamillo, F., & Figueiredo, R. M. (2023). Bioactive limonoids from Carapa guianensis seeds oil and the sustainable use of its byproducts. Current Research in Toxicology, 4, 100–104. https://doi.org/10.1016/j.crtox.2023.100104 +Sistema Nacional de Unidades de Conservação da Natureza (SNUC). (2000). Lei nº 9.985, de 18 de julho de 2000. Diário Oficial da União. +Spruce, R. (1871). On the vegetation of the Amazon Valley as compared with that of the temperate zone. Journal of the Linnean Society of London, Botany, 11(51), 463–482. +Ter Braak, C. J. F. (1986). Canonical correspondence analysis: A new eigenvector technique for multivariate direct gradient analysis. Ecology, 67(5), 1167–1179. https://doi.org/10.2307/1938672 +Ter Braak, C. J. F. (1987). The analysis of vegetation–environment relationships by canonical correspondence analysis. Vegetatio, 69, 69–77. https://doi.org/10.1007/BF00038688 +Ter Steege, H., Pitman, N. C., Sabatier, D., … (2013). Hyperdominance in the Amazonian tree flora. Science, 342(6156), 1243092. https://doi.org/10.1126/science.1243092 +Ter Steege, H., Prado, H., Lima, P. I., … (2020). Biascorrected richness estimates for the Amazonian tree flora. Scientific Reports, 10, 1–13. https://doi.org/10.1038/s41598-020-66686-3 +Therneau, T., Atkinson, B., Ripley, B., & Ripley, M. B. (2017). rpart: Recursive partitioning and regression trees [R package]. Retrieved June 30, 2023, from https://CRAN.R-project.org/package=rpart +Virapongse, A., Endress, B. A., Gilmore, M. P., … (2017). Ecology, livelihoods, and management of the Mauritia flexuosa palm in South America. Global Ecology and Conservation, 10, 70–92. https://doi.org/10.1016/j.gecco.2016.12.005 +Vitousek, P. (1984). Litterfall, nutrient cycling, and nutrient limitation in tropical forests. Ecology, 65(1), 285–298. +Vitousek, P., & Sanford, R. (1986). Nutrient cycling in moist tropical forest. Annual Review of Ecology and Systematics, 17, 137–167. +Wickham, H. (2016). ggplot2: Elegant graphics for data analysis. Springer. +Wittmann, F., Anhuf, A., & Junk, W. J. (2002). Tree species distribution and community structure of central Amazonian várzea forests by remotesensing techniques. Journal of Tropical Ecology, 18(6), 805–820. https://doi.org/10.1017/S0266467402002523 +Wittmann, F., & Junk, W. J. (2003). Sapling communities in Amazonian whitewater forests. Journal of Biogeography, 30(10), 1533–1544. https://doi.org/10.1046/j.1365-2699.2003.00966.x +Wittmann, F., Schöngart, J., Monteiro, J. C., Motzer, T., Junk, W. J., Piedade, M. T. F., Queiroz, H. L., & Worbes, M. (2006). Tree species composition and diversity gradients in whitewater forests across the Amazon Basin. Journal of Biogeography, 33(8), 1334–1347. https://doi.org/10.1111/j.1365-2699.2006.01495.x +Wittmann, F., Schöngart, J., Brito, J. M., Wittmann, A. O., Piedade, M. T. F., Parolin, P., Junk, W. J., & Guillaumet, J. J. (2010). Manual of trees from Central Amazonian várzea floodplains: Taxonomy, ecology and use. INPA. +Wittmann, F., Anhuf, D., Junk, W. J., & Piedade, M. T. F. (2013). Biogeography and biodiversity of Amazonian palm swamps. In W. J. Junk, M. T. F. Piedade, F. Wittmann, J. Schöngart, & P. Parolin (Eds.), Amazonian floodplain forests: Ecophysiology, biodiversity and sustainable management (pp. 355–372). Springer. +Bahrami A, Hasanzadeh M, Hassanian SM, ShahidSales S, Ghayour-Mobarhan M, Ferns GA and Avan A (2017) The potential value of the PI3K/Akt/mTOR signaling pathway for assessing prognosis in cervical cancer and as a target for therapy. J Cell Biochem 118:4163–4169. +Bosch FX, Lorincz A, Muñoz N, Meijer CJLM and Shah K V. (2002) The causal relation between human papillomavirus and cervical cancer. J Clin Pathol 55:244. +Nassu, M. P., Thyssen, P. J., Linhares, A. X. 2014. Developmental rate of immatures of two fly species of forensic importance: Sarcophaga (Liopygia) ruficornis and Microcerella halli (Diptera: Sarcophagidae). Parasitol. Res. 113(1), 217-222. https://doi.org/10.1007/s00436-013-3646-2 +Newton, A. 2022. StaphBase (version Aug 2022). In: O. Bánki et al. (eds), Catalogue of Life (2025-10-10 XR). Amsterdam: Catalogue of Life Foundation, 2025. +Pires, S. M., Cárcamo, M. C., Zimmer, C. R., Ribeiro, P. B. 2009. Influence of diet on the development and reproductive investment of Chrysomya megacephala (Fabricius, 1794) (Diptera: Calliphoridae). Arq. Inst. Biol. (Sao Paulo). 76, 41-47. https://doi.org/10.1590/1808-1657v76p0412009 diff --git a/reference/fixtures/references.xml b/reference/fixtures/references.xml new file mode 100644 index 0000000..476a99c --- /dev/null +++ b/reference/fixtures/references.xml @@ -0,0 +1,2648 @@ + + References + 2.Hou, Y., Chen, L., Li, Z., Zhao, G. and Zhang, C., (2020). Effects of artificial aging on microstructure, mechanical properties and stress corrosion cracking of a novel high strength 7A99 Al alloy. Materials Science and Engineering: A, 780, 139217.HouYChenLLiZZhaoGZhangC2020Effects of artificial aging on microstructure, mechanical properties and stress corrosion cracking of a novel high strength 7A99 Al alloyMaterials Science and Engineering: A78013921723.He, X., Pan, Q., Li, H., Huang, Z., Liu, S., Li, K. and Li, X., (2019). Effect of artificial aging, delayed aging, and pre-aging on microstructure and properties of 6082 aluminum alloy. Metals, 9(2), 173.HeXPanQLiHHuangZLiuSLiKLiX2019Effect of artificial aging, delayed aging, and pre-aging on microstructure and properties of 6082 aluminum alloyMetals9217317324.Mukund, A., Nair, A.S., Nived, S., Raagavendran, R., Premkumar, A., Raj, A.N. and Shankar, K.V., (2021). Impact of solutionising temperature on the microstructure, hardness and tensile strength of Al-6.6 Si-0.3 Mg-3Ni alloys. Materials Today: Proceedings, 38, 2117-2122.MukundANairA.SNivedSRaagavendranRPremkumarARajA.NShankarK.V2021Impact of solutionising temperature on the microstructure, hardness and tensile strength of Al-6.6 Si-0.3 Mg-3Ni alloysMaterials Today: Proceedings3821172122 + AB’SABER, A.N. 2003. Os domínios da natureza no Brasil: + Potencialidades paisagísticas. Ateliê Editorial. + + + + AB’SABER + A.N + + + 2003 + Os domínios da natureza no Brasil: Potencialidades + paisagísticas + Ateliê Editorial + + + + ALVARES, C.A., STAPE, J.L., SENTELHAS, P.C. & GONÇALVES, J.L.M. + 2013a. Modeling monthly mean air temperature for Brazil. Theoretical and Applied + Climatology 113:407–427. + https://doi.org/10.1007/s00704-012-0796-6 + + + + ALVARES + C.A + + + STAPE + J.L + + + SENTELHAS + P.C + + + GONÇALVES + J.L.M + + + 2013a + Modeling monthly mean air temperature for Brazil + Theoretical and Applied Climatology + 113 + 407 + 427 + 10.1007/s00704-012-0796-6 + + + + ALVARES, C.A., STAPE, J.L., SENTELHAS, P.C., GONÇALVES, J.L.M. & + SPAROVEK, G. 2013b. Köppen’s climate classification map for Brazil. + Meteorologische Zeitschrift 22(6):711–728. + https://doi.org/10.1127/0941-2948/2013/0507 + + + + ALVARES + C.A + + + STAPE + J.L + + + SENTELHAS + P.C + + + GONÇALVES + J.L.M + + + SPAROVEK + G + + + 2013b + Köppen’s climate classification map for Brazil + Meteorologische Zeitschrift + 22 + 6 + 711 + 728 + 10.1127/0941-2948/2013/0507 + + + + ANDERSON, J.M. & INGRAM, J.S.I. 1993. Tropical soil + biology and fertility: A handbook of methods (2nd ed.). CAB + International. + + + + ANDERSON + J.M + + + INGRAM + J.S.I + + + 1993 + Tropical soil biology and fertility: A handbook of methods (2nd + ed.) + CAB International + + + + ANDERSON, A.B. & PIRES, J.M. 1978. Study of the buriti palm + (Mauritia flexuosa) in the Amazon and its ecological + significance. Acta Amazonica 8(4):493–502. + + + + ANDERSON + A.B + + + PIRES + J.M + + + 1978 + Study of the buriti palm (Mauritia flexuosa) in the Amazon and + its ecological significance + Acta Amazonica + 8 + 4 + 493 + 502 + + + + ANDRADE, T.M., ASSIS, R.L., WITTMANN, F., SCHÖNGART, J. & + PIEDADE, M.T.F. 2008. Padrões de regeneração em clareiras de origem antrópica na + várzea da RDS Mamirauá, Amazônia Central. Uakari 4(2):19–32. + + + + ANDRADE + T.M + + + ASSIS + R.L + + + WITTMANN + F + + + SCHÖNGART + J + + + PIEDADE + M.T.F + + + 2008 + Padrões de regeneração em clareiras de origem antrópica na várzea + da RDS Mamirauá, Amazônia Central + Uakari + 4 + 2 + 19 + 32 + + + + ASSIS, R.L., WITTMANN, F., PIEDADE, M.T.F., … 2015. Effects of + hydroperiod and substrate properties on tree alpha diversity and composition in + Amazonian floodplain forests. Plant Ecology 216:41–54. + https://doi.org/10.1007/s11258-014-0415-y + + + + ASSIS + R.L + + + WITTMANN + F + + + PIEDADE + M.T.F + + + 2015 + Effects of hydroperiod and substrate properties on tree alpha + diversity and composition in Amazonian floodplain forests + Plant Ecology + 216 + 41 + 54 + 10.1007/s11258-014-0415-y + + + + ASSIS, R.L., WITTMANN, F., BREDIN, Y.K., … 2019. Above-ground woody + biomass distribution in Amazonian floodplain forests: Effects of hydroperiod and + substrate properties. Forest Ecology and Management 432:365–375. + https://doi.org/10.1016/j.foreco.2018.09.031 + + + + ASSIS + R.L + + + WITTMANN + F + + + BREDIN + Y.K + + + 2019 + Above-ground woody biomass distribution in Amazonian floodplain + forests: Effects of hydroperiod and substrate properties + Forest Ecology and Management + 432 + 365 + 375 + 10.1016/j.foreco.2018.09.031 + + + + AUGIE, B. 2017. gridExtra: Miscellaneous functions for + “Grid” graphics (Version 2.3) [R package]. https://CRAN.R-project.org/package=gridExtra + + + + + AUGIE + B + + + 2017 + gridExtra: Miscellaneous functions for “Grid” graphics (Version 2.3) [R + package] + https://CRAN.R-project.org/package=gridExtra + + + + AYRES, J.M.C. 1995. As matas de várzea do Mamirauá. + Sociedade Civil Mamirauá. + + + + AYRES + J.M.C + + + 1995 + As matas de várzea do Mamirauá + Sociedade Civil Mamirauá + + + + BRASIL. 2024. Decreto nº 91.886, de 05 de novembro de 1985. + Diário Oficial da União. https://www.planalto.gov.br/cCivil_03/Atos/decretos/1985/D91886 + + + + Brasil + + 2024 + Decreto nº 91.886, de 05 de novembro de 1985. Diário Oficial da + União + https://www.planalto.gov.br/cCivil_03/Atos/decretos/1985/D91886 + + + + BATISTA, E.S. & CUNHA, C.N. 2012. Structure and floristic + composition of Mauritia flexuosa palm swamps + (buritizais) in the Pantanal, Brazil. Acta Botanica + Brasilica, 26(3):539–552. + + + + BATISTA + E.S + + + CUNHA + C.N + + + 2012 + Structure and floristic composition of Mauritia flexuosa palm + swamps (buritizais) in the Pantanal, Brazil + Acta Botanica Brasilica + 26 + 3 + 539 + 552 + + + + BRIGHTSMITH, D.J. & BRAVO, A. 2006. Ecology and management of + nesting blue-and-yellow macaws (Ara ararauna) in + Mauritia palm swamps. Biodiversity and Conservation + 15(12):4271–4287. https://doi.org/10.1007/s10531-005-3579-x + + + + BRIGHTSMITH + D.J + + + BRAVO + A + + + 2006 + Ecology and management of nesting blue-and-yellow macaws (Ara + ararauna) in Mauritia palm swamps + Biodiversity and Conservation + 15 + 12 + 4271 + 4287 + 10.1007/s10531-005-3579-x + + + + BRITO, J.M., WITTMANN, F., SCHÖNGART, J., … 2008. Guia de 42 + espécies madeireiras da Reserva de Desenvolvimento Sustentável + Mamirauá. Sociedade Civil Mamirauá. + + + + BRITO + J.M + + + WITTMANN + F + + + SCHÖNGART + J + + + 2008 + Guia de 42 espécies madeireiras da Reserva de Desenvolvimento + Sustentável Mamirauá + Sociedade Civil Mamirauá + + + + BROWER, J.E. & ZAR, J.H. 1984. Field and laboratory + methods for general ecology (3rd ed.). WMC Brown + Publishers. + + + + BROWER + J.E + + + ZAR + J.H + + + 1984 + Field and laboratory methods for general ecology (3rd + ed.) + WMC Brown Publishers + + + + CASANOVA, M.T. & BROCK, M.A. 2000. How do depth, duration and + frequency of flooding influence the establishment of wetland plant communities? + Plant Ecology 147(2):237–250. + https://doi.org/10.1023/A:1009875226637 + + + + CASANOVA + M.T + + + BROCK + M.A + + + 2000 + How do depth, duration and frequency of flooding influence the + establishment of wetland plant communities + Plant Ecology + 147 + 2 + 237 + 250 + 10.1023/A:1009875226637 + + + + CARDOSO, D.T., SÄRKINEN, T., ALEXANDER, S., … 2017. Amazon plant + diversity revealed by a taxonomically verified species list. Proceedings of the + National Academy of Sciences 114(40):10695–10700. + https://doi.org/10.1073/pnas.1706756114 + + + + CARDOSO + D.T + + + SÄRKINEN + T + + + ALEXANDER + S + + + 2017 + Amazon plant diversity revealed by a taxonomically verified + species list + Proceedings of the National Academy of Sciences + 114 + 40 + 10695 + 10700 + 10.1073/pnas.1706756114 + + + + CONNELL, J.H. & LOWMAN, M.D. 1989. Low-density tropical rain + forests: Some possible mechanisms for their existence. The American Naturalist + 134:88–119. + + + + CONNELL + J.H + + + LOWMAN + M.D + + + 1989 + Low-density tropical rain forests: Some possible mechanisms for + their existence + The American Naturalist + 134 + 88 + 119 + + + + CORREIA, A.H. & DIRETORIA DE SERVIÇO GEOGRÁFICO DO EXÉRCITO. + 2011. Metodologias e resultados preliminares do Projeto Radiografia da Amazônia. + Anais do XV Simpósio Brasileiro de Sensoriamento + Remoto. + + + + CORREIA + A.H. + + DIRETORIA DE SERVIÇO GEOGRÁFICO DO EXÉRCITO + + 2011 + Metodologias e resultados preliminares do Projeto Radiografia da + Amazônia + Anais do XV Simpósio Brasileiro de Sensoriamento Remoto + + + + DAMBROS, C.S. 2020. csdambros/R-functions: First + release (Version 1.0) [Computer software]. Zenodo. + https://doi.org/10.5281/zenodo.3784397 + + + + DAMBROS + C.S + + + 2020 + csdambros/R-functions: First release (Version 1.0) [Computer + software] + 10.5281/zenodo.3784397 + + + + DINNO, A. 2017. dunn.test: Dunn’s test of multiple + comparisons using rank sums (Version 1.3.5) [R package]. https://CRAN.R-project.org/package=dunn.test + + + + + DINNO + A + + + 2017 + dunn.test: Dunn’s test of multiple comparisons using rank sums (Version + 1.3.5) [R package] + https://CRAN.R-project.org/package=dunn.test + + + + DURGANTE, F.M., HIGUCHI, N., OHASHI, S., … 2023. Soil fertility and + drought interact to determine large variations in wood production for a + hyperdominant Amazonian tree species. Frontiers in Forests and Global Change, 5, + Article 1065645. https://doi.org/10.3389/ffgc.2022.1065645 + + + + DURGANTE + F.M + + + HIGUCHI + N + + + OHASHI + S + + + 2023 + Soil fertility and drought interact to determine large variations + in wood production for a hyperdominant Amazonian tree + species + Frontiers in Forests and Global Change, 5, Article 1065645 + 10.3389/ffgc.2022.1065645 + + + + EMILIO, T., NELSON, B.W. & PRADO, P.I. 2013. Unexpected + monoculture in a hyperdiverse tropical forest. Scientific Reports 3:3267. + https://doi.org/10.1038/srep03267 + + + + EMILIO + T + + + NELSON + B.W + + + PRADO + P.I + + + 2013 + Unexpected monoculture in a hyperdiverse tropical + forest + Scientific Reports + 3:3267 + 10.1038/srep03267 + + + + ENGELBRECHT, B.M., VÁSQUEZ-RIVERA, T., TYREE, M.T., KURSAR, T.A. + & CHAMBI-SALAZAR, A. 2007. Drought effects on seedling survival in a + tropical moist forest. Trees 21(5):389–398. + https://doi.org/10.1007/s00468-004-0393-0 + + + + ENGELBRECHT + B.M + + + VÁSQUEZ-RIVERA + T + + + TYREE + M.T + + + KURSAR + T.A + + + CHAMBI-SALAZAR + A + + + 2007 + Drought effects on seedling survival in a tropical moist + forest + Trees + 21 + 5 + 389 + 398 + 10.1007/s00468-004-0393-0 + + + + FELIX RIBEIRO, K.A. 2025. Replication data for: Mauritia + flexuosa (Buriti) forest description: Hydro-edaphic effects on tree + species distribution in Western Amazon. SciELO Data. Draft version. + https://doi.org/10.48331/SCIELODATA.RIVAW4 + + + + FELIX RIBEIRO + K.A + + + 2025 + Replication data for: Mauritia flexuosa (Buriti) forest + description: Hydro-edaphic effects on tree species distribution in Western + Amazon + SciELO Data. Draft version + 10.48331/SCIELODATA.RIVAW4 + + + + FERREIRA, L.V. & STOHLGREN, T.J. 1999. Effects of river level + fluctuation on plant species richness, diversity, and distribution in a + floodplain forest in Central Amazonia. Oecologia + 120(4):582–587. + + + + FERREIRA + L.V + + + STOHLGREN + T.J + + + 1999 + Effects of river level fluctuation on plant species richness, + diversity, and distribution in a floodplain forest in Central + Amazonia + Oecologia + 120 + 4 + 582 + 587 + + + + FLEISCHMANN, A.S., ALVES, P.C. & NASCIMENTO, A.Z. 2023. + Caracterização geográfica da Área de Relevante Interesse Ecológico + Javari-Buriti. In Instituto de Desenvolvimento Sustentável Mamirauá (Ed.), + Projeto Javari-Buriti: Subsídios para a elaboração do Plano de + Manejo da Área de Relevante Interesse Ecológica Javari-Buriti (pp. + 15–48). Instituto de Desenvolvimento Sustentável Mamirauá. + + + + FLEISCHMANN + A.S + + + ALVES + P.C + + + NASCIMENTO + A.Z + + + 2023 + Caracterização geográfica da Área de Relevante Interesse + Ecológico Javari-Buriti + In Instituto de Desenvolvimento Sustentável Mamirauá (Ed.), Projeto + Javari-Buriti: Subsídios para a elaboração do Plano de Manejo da Área de + Relevante Interesse Ecológica Javari-Buriti (pp. 15–48). Instituto de + Desenvolvimento Sustentável Mamirauá + + + + FLORA E FUNGA DO BRASIL. 2022. Jardim Botânico do Rio de + Janeiro. Retrieved December 19, 2022, from http://floradobrasil.jbrj.gov.br + + + + FLORA E FUNGA DO BRASIL + + 2022 + Jardim Botânico do Rio de Janeiro. Retrieved December + 19, 2022 + + + + GALEANO, A., URREGO, L.E., SÁNCHEZ, M., … 2015. Environmental + drivers for regeneration of Mauritia flexuosa Lf in Colombian + Amazonian swamp forest. Aquatic Botany 123:47–53. + https://doi.org/10.1016/j.aquabot.2015.02.001 + + + + GALEANO + A + + + URREGO + L.E + + + SÁNCHEZ + M + + + 2015 + Environmental drivers for regeneration of Mauritia flexuosa Lf in + Colombian Amazonian swamp forest + Aquatic Botany + 123 + 47 + 53 + 10.1016/j.aquabot.2015.02.001 + + + + GRAHAM, J. 2003. HH: Regression and other stories + [R package]. https://CRAN.R-project.org/package=HH + + + + + GRAHAM + J + + + 2003 + HH: Regression and other stories [R package] + https://CRAN.R-project.org/package=HH + + + + GOTELLI, N.J. & COLWELL, R.K. 2001. Quantifying biodiversity: + Procedures and pitfalls in the measurement and comparison of species richness. + Ecology Letters 4:379–391. + https://doi.org/10.1046/j.1461-0248.2001.00230.x + + + + GOTELLI + N.J + + + COLWELL + R.K + + + 2001 + Quantifying biodiversity: Procedures and pitfalls in the + measurement and comparison of species richness + Ecology Letters + 4 + 379 + 391 + 10.1046/j.1461-0248.2001.00230.x + + + + HAMILTON, S.K., SIPPEL, S.J. & MELACK, J.M. 2007. Seasonal + inundation patterns in two large savanna floodplains of South America: The + Llanos de Moxos (Bolivia) and the Llanos del Orinoco (Venezuela and Colombia). + Hydrological Processes 21(14):1719–1736. + https://doi.org/10.1002/hyp.5559 + + + + HAMILTON + S.K + + + SIPPEL + S.J + + + MELACK + J.M + + + 2007 + Seasonal inundation patterns in two large savanna floodplains of + South America: The Llanos de Moxos (Bolivia) and the Llanos del Orinoco + (Venezuela and Colombia) + Hydrological Processes + 21 + 14 + 1719 + 1736 + 10.1002/hyp.5559 + + + + HART, T.B., HART, J.A. & MURPHY, P.G. 1989. Monodominant and + species-rich forests of the humid tropics: Causes for their co-occurrence. The + American Naturalist 133(5):613–633. + + + + HART + T.B + + + HART + J.A + + + MURPHY + P.G + + + 1989 + Monodominant and species-rich forests of the humid tropics: + Causes for their co-occurrence + The American Naturalist + 133 + 5 + 613 + 633 + + + + HESS, L.L. 2003. Mapping wetlands in the Amazon Basin using SAR + data. In Proceedings of the IEEE International Geoscience and Remote + Sensing Symposium (IGARSS ’03) (Vol. 2, pp. + 1375–1377). IEEE. + + + + HESS + L.L + + + 2003 + Mapping wetlands in the Amazon Basin using SAR + data + In Proceedings of the IEEE International Geoscience and Remote Sensing + Symposium (IGARSS ’03) + 2 + 1375 + 1377 + IEEE + + + + HOUSEHOLDER, J.E., SCHUMANN, D.A. & HAMILTON, S.K. 2010. + Influence of hydrologic connectivity and environmental heterogeneity on + floodplain wetland vegetation patterns in the Amazon River basin. Wetlands + 30(6):1167–1180. + + + + HOUSEHOLDER + J.E + + + SCHUMANN + D.A + + + HAMILTON + S.K + + + 2010 + Influence of hydrologic connectivity and environmental + heterogeneity on floodplain wetland vegetation patterns in the Amazon River + basin + Wetlands + 30 + 6 + 1167 + 1180 + + + + HOUSEHOLDER, J.E., SCHUMANN, D.A. & HAMILTON, S.K. 2012. + Assessing inundation impacts in floodplain wetlands: Relations between flood + duration and plant community composition. Wetlands + 32(6):1091–1103. + + + + HOUSEHOLDER + J.E + + + SCHUMANN + D.A + + + HAMILTON + S.K + + + 2012 + Assessing inundation impacts in floodplain wetlands: Relations + between flood duration and plant community composition + Wetlands + 32 + 6 + 1091 + 1103 + + + + INSTITUTO BRASILEIRO DE GEOGRAFIA E ESTATÍSTICA (IBGE). 2019a. + Índice de Organização do Território / Malhas Territoriais / Malhas + Municipais / Município 2015 / Brasil [Data set]. Retrieved August + 28, 2019, from ftp://geoftp.ibge.gov.br/organizacao_do_territorio/malhas_territoriais/malhas_municipais/municipio_2015/Brasil/BR/ + + + + INSTITUTO BRASILEIRO DE GEOGRAFIA E ESTATÍSTICA (IBGE) + + 2019a + Índice de Organização do Território / Malhas Territoriais / + Malhas Municipais / Município 2015 / Brasil [Data set]. + Retrieved August 28, 2019 + + + + INSTITUTO BRASILEIRO DE GEOGRAFIA E ESTATÍSTICA (IBGE). 2019b. + Bases cartográficas contínuas BC250 (versão 2017) – + Shapefiles [Data set]. Retrieved August 28, 2019, from http://geoftp.ibge.gov.br/cartas_e_mapas/bases_cartograficas_continuas/bc250/versao2017/shapefile/ + + + + INSTITUTO BRASILEIRO DE GEOGRAFIA E ESTATÍSTICA (IBGE) + + 2019b + Bases cartográficas contínuas BC250 (versão 2017) – + Shapefiles [Data set]. Retrieved August 28, 2019 + + + + JUNK, W. J. 1997. General aspects of floodplain ecology with special + reference to Amazonian floodplains. In W.J. Junk (Ed.), The Central + Amazon floodplain: Ecology of a pulsing system (Ecological Studies + 126, pp. 3–20). Springer. + + + + JUNK + W. J + + + 1997 + General aspects of floodplain ecology with special reference to + Amazonian floodplains + + + JUNK + W.J + + Ed + + The Central Amazon floodplain: Ecology of a pulsing system (Ecological + Studies 126 + 3 + 20 + Springer + + + + JUNK, W.J., BAYLEY, P.B. & SPARKS, R.E. 1989. The flood pulse + concept in river–floodplain systems. Canadian Special Publication of Fisheries + and Aquatic Sciences 106(1):110–127. + + + + JUNK + W.J + + + BAYLEY + P.B + + + SPARKS + R.E + + + 1989 + The flood pulse concept in river–floodplain + systems + Canadian Special Publication of Fisheries and Aquatic Sciences + 106 + 1 + 110 + 127 + + + + JUNK, W.J. & FURCH, K. 1993. The physical and chemical limnology + of Amazonian waters and their role in the carbon cycle. In G. T. Prance & T. + E. Lovejoy (Eds.), Amazonia (pp. 385–404). Pergamon + Press. + + + + JUNK + W.J + + + FURCH + K + + + 1993 + The physical and chemical limnology of Amazonian waters and their + role in the carbon cycle + + + PRANCE + G. T + + + LOVEJOY + T. E + + Eds + + Amazonia + 385 + 404 + Pergamon Press + + + + JUNK, W.J., PIEDADE, M.T.F., SCHÖNGART, J., COHN-HAFT, M., ADENEY, + J.M. & WITTMANN, F. 2011. A classification of major naturally occurring + Amazonian lowland wetlands. Wetlands 31(4):623–640. + https://doi.org/10.1007/s13157-011-0190-7 + + + + JUNK + W.J + + + PIEDADE + M.T.F + + + SCHÖNGART + J + + + COHN-HAFT + M + + + ADENEY + J.M + + + WITTMANN + F + + + 2011 + A classification of major naturally occurring Amazonian lowland + wetlands + Wetlands + 31 + 4 + 623 + 640 + 10.1007/s13157-011-0190-7 + + + + JUNK, W.J., PIEDADE, M.T.F., LOURIVAL, R., WITTMANN, F., KANDUS, P., + LACERDA, L.D., BOZELLI, R.L., ESTEVES, F.A., NUNES DA CUNHA, C., MALTCHIK, L., + SCHÖNGART, J., SCHAEFFER-NOVELLI, Y. & AGOSTINHO, A.A. 2014. Brazilian + wetlands: Their definition, delineation, and classification for research, + sustainable management, and protection. Aquatic Conservation: Marine and + Freshwater Ecosystems 24(1):5–22. + https://doi.org/10.1002/aqc.2386 + + + + JUNK + W.J + + + PIEDADE + M.T.F + + + LOURIVAL + R + + + WITTMANN + F + + + KANDUS + P + + + LACERDA + L.D + + + BOZELLI + R.L + + + ESTEVES + F.A + + + NUNES DA CUNHA + C + + + MALTCHIK + L + + + SCHÖNGART + J + + + SCHAEFFER-NOVELLI + Y + + + AGOSTINHO + A.A + + + 2014 + Brazilian wetlands: Their definition, delineation, and + classification for research, sustainable management, and + protection + Aquatic Conservation: Marine and Freshwater Ecosystems + 24 + 1 + 5 + 22 + 10.1002/aqc.2386 + + + + KAHN, F. & MEJIA, K. 1990. Palm communities in wetland forest + ecosystems of Peruvian Amazonia. Forest Ecology and Management 33–34:169–179. + https://doi.org/10.1016/0378-1127(90)90195-X + + + + KAHN + F + + + MEJIA + K + + + 1990 + Palm communities in wetland forest ecosystems of Peruvian + Amazonia + Forest Ecology and Management + 33–34 + 169 + 179 + 10.1016/0378-1127(90)90195-X + + + + KAHN, F. & DE GRANVILLE, J.-J. 1992. Palms in forest + ecosystems of Amazonia. Springer-Verlag. + + + + KAHN + F + + + DE GRANVILLE + J.-J + + + 1992 + Palms in forest ecosystems of Amazonia + Springer-Verlag + + + + KELLY, T.J., BAIRD, A.J., ROUCOUX, K.H., BAKER, T.R., + HONORIO-CORONADO, E.M., RÍOS, M. & LAWSON, I.T. 2014. The high hydraulic + conductivity of three wooded tropical peat swamps in northeast Peru: + Measurements and implications for hydrological function. Hydrological Processes + 28:3373–3387. https://doi.org/10.1002/hyp.9884 + + + + KELLY + T.J + + + BAIRD + A.J + + + ROUCOUX + K.H + + + BAKER + T.R + + + HONORIO-CORONADO + E.M + + + RÍOS + M + + + LAWSON + I.T + + + 2014 + The high hydraulic conductivity of three wooded tropical peat + swamps in northeast Peru: Measurements and implications for hydrological + function + Hydrological Processes + 28 + 3373 + 3387 + 10.1002/hyp.9884 + + + + KLIMAS, C.A., KAINER, K.A., OLIVEIRA, W. & LÚCIA, H. 2012. The + economic value of sustainable seed and timber harvests of multi-use species: An + example using Carapa guianensis. Forest Ecology and Management 268:81–91. + https://doi.org/10.1016/j.foreco.2011.03.006 + + + + KLIMAS + C.A + + + KAINER + K.A + + + OLIVEIRA + W + + + LÚCIA + H + + + 2012 + The economic value of sustainable seed and timber harvests of + multi-use species: An example using Carapa guianensis + Forest Ecology and Management + 268 + 81 + 91 + 10.1016/j.foreco.2011.03.006 + + + + KUBITZKI, K. 1989. The ecology of Amazonian palm swamps. In K. + Kubitzki (Ed.), Tropical rain forest ecosystems: Biogeographical and + ecological studies (pp. 227–239). Springer. + + + + KUBITZKI + K + + + 1989 + The ecology of Amazonian palm swamps + + + KUBITZKI + K + + + Ed + Tropical rain forest ecosystems: Biogeographical and ecological + studies + 227 + 239 + Springer + + + + LÄHTEENOJA, O., RUOKOLAINEN, K., SCHULMAN, L., … 2012. Amazonian + peatlands: An ignored C sink and potential source. Global Change Biology + 18(12):3366–3376. + https://doi.org/10.1111/j.1365-2486.2009.01920.x + + + + LÄHTEENOJA + O + + + RUOKOLAINEN + K + + + SCHULMAN + L + + + 2012 + Amazonian peatlands: An ignored C sink and potential + source + Global Change Biology + 18 + 12 + 3366 + 3376 + 10.1111/j.1365-2486.2009.01920.x + + + + LORENZI, H. 2002. Árvores brasileiras: Manual de + identificação e cultivo de plantas arbóreas nativas do Brasil (Vol. + 1). Editora Plantarum. + + + + LORENZI + H + + + 2002 + Árvores brasileiras: Manual de identificação e cultivo de plantas + arbóreas nativas do Brasil (Vol. 1) + Editora Plantarum + + + + MAGURRAN, A.E. 1988. Ecological diversity and its + measurement. Princeton University Press. + + + + MAGURRAN + A.E + + + 1988 + Ecological diversity and its measurement + Princeton University Press + + + + MAGNUSSEN, S., KLEINN, C. & PICARD, N. 2001. A review of the + concept of cluster sampling in forest inventories. Canadian Journal of Forest + Research 31(7):1105–1115. https://doi.org/10.1139/x01-054 + + + + MAGNUSSEN + S + + + KLEINN + C + + + PICARD + N + + + 2001 + A review of the concept of cluster sampling in forest + inventories + Canadian Journal of Forest Research + 31 + 7 + 1105 + 1115 + 10.1139/x01-054 + + + + MARSHALL, B.G., FORSBERG, B.R., HESS, L.L. & CARVALHO-FREITAS, + C.E. 2011. Water temperature differences in interfluvial palm swamp habitats of + Paracheirodon axelrodi and P. simulans + (Osteichthyes: Characidae) in the middle Rio Negro, Brazil. Ichthyological + Exploration of Freshwaters 22(4):377–383. + + + + MARSHALL + B.G + + + FORSBERG + B.R + + + HESS + L.L + + + CARVALHO-FREITAS + C.E + + + 2011 + Water temperature differences in interfluvial palm swamp habitats + of Paracheirodon axelrodi and P. simulans (Osteichthyes: Characidae) in the + middle Rio Negro, Brazil + Ichthyological Exploration of Freshwaters + 22 + 4 + 377 + 383 + + + + MENDIBURU, F. 2023. agricolae: Statistical procedures for + agricultural research (Version 1.3-7) [R package]. https://CRAN.R-project.org/package=agricolae + + + + + MENDIBURU + F + + + 2023 + agricolae: Statistical procedures for agricultural research (Version + 1.3-7) [R package] + https://CRAN.R-project.org/package=agricolae + + + + MILBORROW, S. 2018. rpart.plot: Plot “rpart” models—An + enhanced version of plot.rpart [R package]. Retrieved June 30, + 2023, from https://CRAN.R-project.org/package=rpart.plot + + + + + MILBORROW + S + + + 2018 + + + + + MOREIRA, D.M. 2016. Geodésia aplicada ao monitoramento + hidrológico da bacia Amazônica [Doctoral dissertation, Universidade + Federal do Rio de Janeiro]. + + + + MOREIRA + D.M + + + 2016 + Geodésia aplicada ao monitoramento hidrológico da bacia + Amazônica + [Doctoral dissertation, Universidade Federal do Rio de + Janeiro]. + + + + MUELLER-DOMBOIS, D. & ELLENBERG, H. 1974. Aims and + methods of vegetation ecology. John Wiley & + Sons. + + + + MUELLER-DOMBOIS + D + + + ELLENBERG + H + + + 1974 + Aims and methods of vegetation ecology + John Wiley & Sons + + + + OLIVEIRA, R.L.C., RICARDO, P., VERIDIANA, S. & BARBOSA, R.I. + 2017. Structure and tree species composition in different habitats of savanna + used by indigenous people in the Northern Brazilian Amazon. Biodiversity Data + Journal, 5, Article e20044. + https://doi.org/10.3897/BDJ.5.e20044 + + + + OLIVEIRA + R.L.C + + + RICARDO + P + + + VERIDIANA + S + + + BARBOSA + R.I + + + 2017 + Structure and tree species composition in different habitats of + savanna used by indigenous people in the Northern Brazilian + Amazon + Biodiversity Data Journal, 5, Article + e2004410.3897/BDJ.5.e20044 + + + + OLIVEIRA, R.L.C., SCUDELLER, V.V., BARBOSA, R.I., … 2019. Aparência + ecológica e conservação de espécies lenhosas pelos Makuxis nas savanas de + Roraima, Amazônia brasileira. Ethnoscientia – Brazilian Journal of + Ethnobiology and Ethnoecology 4:1–14. http://dx.doi.org/10.18542/ethnoscientia.v0i0.10245 + + + + + OLIVEIRA + R.L.C + + + SCUDELLER + V.V + + + BARBOSA + R.I + + + 2019 + Aparência ecológica e conservação de espécies lenhosas pelos + Makuxis nas savanas de Roraima, Amazônia brasileira + Ethnoscientia – Brazilian Journal of Ethnobiology and + Ethnoecology + 4 + 1 + 14 + http://dx.doi.org/10.18542/ethnoscientia.v0i0.10245 + + + + OKSANEN, J., BLANCHET, F.G., FRIENDLY, M., … 2018. vegan: + Community ecology package (Version 2.5-3) [R package]. Retrieved + June 30, 2023, from https://CRAN.R-project.org/package=vegan + + + + + OKSANEN + J + + + BLANCHET + F.G + + + FRIENDLY + M + + + 2018 + + + + + OKSANEN, J., BLANCHET, F.G., FRIENDLY, M., … 2022. vegan: + Community ecology package (Version 2.6-4) [R package]. Retrieved + June 30, 2023, from https://cran.r-project.org/package=vegan + + + + + OKSANEN + J + + + BLANCHET + F.G + + + FRIENDLY + M + + + 2022 + + + + + OLIVEIRA, N.A. & AMARAL, I.L. 2005. Aspectos florísticos, + fitossociológicos e ecológicos de um sub-bosque de terra firme na Amazônia + Central, Amazonas, Brasil. Acta Amazônica 35(1):1–16. + https://doi.org/10.1590/S0044-59672005000100002 + + + + OLIVEIRA + N.A + + + AMARAL + I.L + + + 2005 + Aspectos florísticos, fitossociológicos e ecológicos de um + sub-bosque de terra firme na Amazônia Central, Amazonas, + Brasil + Acta Amazônica + 35 + 1 + 1 + 16 + 10.1590/S0044-59672005000100002 + + + + OYELADE, J.O., IDOWU, D.O., ONIYA, O.O. & OGUNKUNLE, D.O. 2017. + Optimization of biodiesel production from sandbox (Hura crepitans L.) seed oil + using two different catalysts. Energy Sources, Part A: Recovery, Utilization, + and Environmental Effects 39(12):1242–1249. + https://doi.org/10.1080/15567036.2017.1320691 + + + + OYELADE + J.O + + + IDOWU + D.O + + + ONIYA + O.O + + + OGUNKUNLE + D.O + + + 2017 + Optimization of biodiesel production from sandbox (Hura crepitans + L.) seed oil using two different catalysts + Energy Sources, Part A: Recovery, Utilization, and Environmental + Effects + 39 + 12 + 1242 + 1249 + 10.1080/15567036.2017.1320691 + + + + PELLA, E. 1990. Elemental organic analysis. Part 2. State of the + art. American Laboratory 22:28–32. + + + + PELLA + E + + + 1990 + Elemental organic analysis. Part 2. State of the + art + American Laboratory + 22 + 28 + 32 + + + + PERKINS, J.L. 1982. Shannon-Weaver or Shannon-Wiener. Journal of the + Water Pollution Control Federation 54(7):1049–1050. + + + + PERKINS + J.L + + + 1982 + Shannon-Weaver or Shannon-Wiener + Journal of the Water Pollution Control Federation + 54 + 7 + 1049 + 1050 + + + + PEREIRA, M.G., VALLADARES, G.S., ANJOS, L.H.C., BENITES, V.M., + ESPÍNDULA-JÚNIOR, A. & EBELING, A.G. 2006. Organic carbon determination in + Histosols and soil horizons with high organic matter content from Brazil. + Scientia Agricola 63(2):187–193. + https://doi.org/10.1590/S0103-90162006000200012 + + + + PEREIRA + M.G + + + VALLADARES + G.S + + + ANJOS + L.H.C + + + BENITES + V.M + + + ESPÍNDULA-JÚNIOR + A + + + EBELING + A.G + + + 2006 + Organic carbon determination in Histosols and soil horizons with + high organic matter content from Brazil + Scientia Agricola + 63 + 2 + 187 + 193 + 10.1590/S0103-90162006000200012 + + + + PINTAUD, J.-C., DRANSFIELD, J., HENDERSON, A., BORCHSENIUS, F., + MOGENS, M.B., SALO, J. & BALSLEV, H. 2008. A revision of the palm genera + (Arecaceae). Botanical Journal of the Linnean Society 157(1):1–68. + https://doi.org/10.1111/j.1095-8339.2008.00728.x + + + + PINTAUD + J.-C + + + DRANSFIELD + J + + + HENDERSON + A + + + BORCHSENIUS + F + + + MOGENS + M.B + + + SALO + J + + + BALSLEV + H + + + 2008 + A revision of the palm genera (Arecaceae) + Botanical Journal of the Linnean Society + 157 + 1 + 1 + 68 + 10.1111/j.1095-8339.2008.00728.x + + + + POGGIO, L., DE SOUSA, L.M., BATJES, N.H., HEUVELINK, G.B.M., KEMPEN, + B., RIBEIRO, E. & ROSSITER, D. 2021. SoilGrids 2.0: Producing soil + information for the globe with quantified spatial uncertainty. Soil, + 7:217–240. + + + + POGGIO + L + + + DE SOUSA + L.M + + + BATJES + N.H + + + HEUVELINK + G.B.M + + + KEMPEN + B + + + RIBEIRO + E + + + ROSSITER + D + + + 2021 + SoilGrids 2.0: Producing soil information for the globe with + quantified spatial uncertainty + Soil + 7 + 217 + 240 + + + + QUESADA, C.A., PHILLIPS, O.L., SCHWARZ, M., … 2012. Basin-wide + variations in Amazon forest structure and function are mediated by both soils + and climate. Biogeosciences 9(6):2203–2246. + https://doi.org/10.5194/bg-9-2203-2012 + + + + QUESADA + C.A + + + PHILLIPS + O.L + + + SCHWARZ + M + + + 2012 + Basin-wide variations in Amazon forest structure and function are + mediated by both soils and climate + Biogeosciences + 9 + 6 + 2203 + 2246 + 10.5194/bg-9-2203-2012 + + + + R CORE TEAM. 2021. R: A language and environment for + statistical computing [Computer software]. R Foundation for + Statistical Computing. Retrieved May 14, 2023, from https://www.R-project.org/ + + + + R CORE TEAM + + 2021 + R: A language and environment for statistical computing [Computer + software] + + + + RESENDE, I.L.M., SANTOS, F.P., CHAVES, L.J. & NASCIMENTO, J.L. + 2012. Estrutura etária de populações de Mauritia flexuosa L. + (Arecaceae) de veredas da região central de Goiás, Brasil. Revista Árvore + 36(1):103–112. https://doi.org/10.1590/S0100-67622012000100012 + + + + RESENDE + I.L.M + + + SANTOS + F.P + + + CHAVES + L.J + + + NASCIMENTO + J.L + + + 2012 + Estrutura etária de populações de Mauritia flexuosa L. + (Arecaceae) de veredas da região central de Goiás, Brasil + Revista Árvore + 36 + 1 + 103 + 112 + 10.1590/S0100-67622012000100012 + + + + RIGUEIRA, S., BRINA, A.E., FILHO, J.R., COSTA-SILVA, L.V., BEDÊ, + L.C. & REZENDE, M. 2002. Projeto Buriti: Artesanato, natureza e + sociedade. Instituto Terra Brasilis de Desenvolvimento + Sócio-Ambiental. + + + + RIGUEIRA + S + + + BRINA + A.E + + + FILHO + J.R + + + COSTA-SILVA + L.V + + + BEDÊ + L.C + + + REZENDE + M + + + 2002 + Projeto Buriti: Artesanato, natureza e sociedade + Instituto Terra Brasilis de Desenvolvimento Sócio-Ambiental + + + + ROSA, R., BARBOSA, R.I. & KOPTUR, S. 2014. Which factors explain + reproductive output of Mauritia flexuosa (Arecaceae) in forest + and savanna habitats of northern Amazonia? International Journal of Plant + Sciences 175(3):307–318. https://doi.org/10.1086/674446 + + + + ROSA + R + + + BARBOSA + R.I + + + KOPTUR + S + + + 2014 + Which factors explain reproductive output of Mauritia flexuosa + (Arecaceae) in forest and savanna habitats of northern + Amazonia + International Journal of Plant Sciences + 175 + 3 + 307 + 318 + 10.1086/674446 + + + + SALINAS, H. & RAMÍREZ-DELGADO, D. 2021. ecolTest: + Community ecology tests [R package]. + + + + SALINAS + H + + + RAMÍREZ-DELGADO + D + + + 2021 + ecolTest: Community ecology tests + + + + SANDER, N.L. 2014. Estrutura, composição florística e + etnobiologia de um buritizal na fronteira biológica + Amazônia-Cerrado [Master’s thesis, Universidade do Estado de Mato + Grosso]. + + + + SANDER + N.L + + + 2014 + Estrutura, composição florística e etnobiologia de um buritizal na + fronteira biológica Amazônia-Cerrado + [Master’s thesis, Universidade do Estado de Mato + Grosso] + + + + SANDER, N.L., RIBEIRO, R.S., SILVA, D.R., … 2017. Floristic, + phytosociology and spatial distribution of a monodominant Mauritia + flexuosa L. f. forest in a Southern Amazon in the Arc of + deforestation. In Natural resources in wetlands: From Pantanal to + Amazonia (pp. – ). MPEG. + + + + SANDER + N.L + + + RIBEIRO + R.S + + + SILVA + D.R + + + 2017 + Floristic, phytosociology and spatial distribution of a + monodominant Mauritia flexuosa L. f. forest in a Southern Amazon in the Arc + of deforestation + In Natural resources in wetlands: From Pantanal to Amazonia (pp. – + ) + MPEG + + + + SANQUETTA, C.R., CORTE, A.P.D. & NETTO, S.P. 2014. + Inventários florestais: Planejamento e execução. + Universidade Federal do Paraná (UFPR). + + + + SANQUETTA + C.R + + + CORTE + A.P.D + + + NETTO + S.P + + + 2014 + Inventários florestais: Planejamento e execução + Universidade Federal do Paraná (UFPR) + + + + SCHAAP, K.J., FUCHSLUEGER, L., HOOSBEEK, M.R., HOFHANSL, F., + MARTINS, N.P., VALVERDE-BARRANTES, O.J. & QUESADA, C.A. 2021. Litter inputs + and phosphatase activity affect the temporal variability of organic phosphorus + in a tropical forest soil in the Central Amazon. Plant and Soil + 469(1–2):423–441. + + + + SCHAAP + K.J + + + FUCHSLUEGER + L + + + HOOSBEEK + M.R + + + HOFHANSL + F + + + MARTINS + N.P + + + VALVERDE-BARRANTES + O.J + + + QUESADA + C.A + + + 2021 + Litter inputs and phosphatase activity affect the temporal + variability of organic phosphorus in a tropical forest soil in the Central + Amazon + Plant and Soil + 469 + 1–2 + 423 + 441 + + + + SILVA, V.P., CARVALHO-BRITO, L., MARQUES, A.M., CUNHA-CAMILLO, F. + & FIGUEIREDO, R.M. 2023. Bioactive limonoids from Carapa + guianensis seeds oil and the sustainable use of its by-products. + Current Research in Toxicology 4:100–104. + https://doi.org/10.1016/j.crtox.2023.100104 + + + + SILVA + V.P + + + CARVALHO-BRITO + L + + + MARQUES + A.M + + + CUNHA-CAMILLO + F + + + FIGUEIREDO + R.M + + + 2023 + Bioactive limonoids from Carapa guianensis seeds oil and the + sustainable use of its by-products + Current Research in Toxicology + 4 + 100 + 104 + 10.1016/j.crtox.2023.100104 + + + + SISTEMA NACIONAL DE UNIDADES DE CONSERVAÇÃO DA NATUREZA (SNUC). + 2000. Lei nº 9.985, de 18 de julho de 2000. Diário Oficial da + União. + + + SISTEMA NACIONAL DE UNIDADES DE CONSERVAÇÃO DA NATUREZA + (SNUC). + + 2000 + Lei nº 9.985, de 18 de julho de 2000 + + + + SPRUCE, R. 1871. On the vegetation of the Amazon Valley as compared + with that of the temperate zone. Journal of the Linnean Society of London, + Botany 11(51):463–482. + + + + SPRUCE + R + + + 1871 + On the vegetation of the Amazon Valley as compared with that of + the temperate zone + Journal of the Linnean Society of London, Botany + 11 + 51 + 463 + 482 + + + + TER BRAAK, C.J.F. 1986. Canonical correspondence analysis: A new + eigenvector technique for multivariate direct gradient analysis. Ecology + 67(5):1167–1179. https://doi.org/10.2307/1938672 + + + + TER BRAAK + C.J.F + + + 1986 + Canonical correspondence analysis: A new eigenvector technique + for multivariate direct gradient analysis + Ecology + 67 + 5 + 1167 + 1179 + 10.2307/1938672 + + + + TER BRAAK, C.J.F. 1987. The analysis of vegetation–environment + relationships by canonical correspondence analysis. Vegetatio 69:69–77. + https://doi.org/10.1007/BF00038688 + + + + TER BRAAK + C.J.F + + + 1987 + The analysis of vegetation–environment relationships by canonical + correspondence analysis + Vegetatio + 69 + 69 + 77 + 10.1007/BF00038688 + + + + TER STEEGE, H., PITMAN, N.C., SABATIER, D., … 2013. Hyperdominance + in the Amazonian tree flora. Science 342(6156):1243092. + https://doi.org/10.1126/science.1243092 + + + + TER STEEGE + H + + + PITMAN + N.C + + + SABATIER + D + + + 2013 + Hyperdominance in the Amazonian tree flora + Science + 342 + 6156 + 124309210.1126/science.1243092 + + + + TER STEEGE, H., PRADO, H., LIMA, P.I., … 2020. Bias-corrected + richness estimates for the Amazonian tree flora. Scientific Reports 10:1–13. + https://doi.org/10.1038/s41598-020-66686-3 + + + + TER STEEGE + H + + + PRADO + H + + + LIMA + P.I + + + 2020 + Bias-corrected richness estimates for the Amazonian tree + flora + Scientific Reports + 10 + 1 + 13 + 10.1038/s41598-020-66686-3 + + + + THERNEAU, T., ATKINSON, B., RIPLEY, B. & RIPLEY, M.B. 2017. + rpart: Recursive partitioning and regression trees [R + package]. Retrieved June 30, 2023, from https://CRAN.R-project.org/package=rpart + + + + + THERNEAU + T + + + ATKINSON + B + + + RIPLEY + B + + + RIPLEY + M.B + + + 2017 + + + + + VIRAPONGSE, A., ENDRESS, B.A., GILMORE, M.P., … 2017. Ecology, + livelihoods, and management of the Mauritia flexuosa palm in + South America. Global Ecology and Conservation 10:70–92. + https://doi.org/10.1016/j.gecco.2016.12.005 + + + + VIRAPONGSE + A + + + ENDRESS + B.A + + + GILMORE + M.P + + + 2017 + Ecology, livelihoods, and management of the Mauritia flexuosa + palm in South America + Global Ecology and Conservation + 10 + 70 + 92 + 10.1016/j.gecco.2016.12.005 + + + + VITOUSEK, P. 1984. Litterfall, nutrient cycling, and nutrient + limitation in tropical forests. Ecology 65(1):285–298. + + + + VITOUSEK + P + + + 1984 + Litterfall, nutrient cycling, and nutrient limitation in tropical + forests + Ecology + 65 + 1 + 285 + 298 + + + + VITOUSEK, P. & SANFORD, R. 1986. Nutrient cycling in moist + tropical forest. Annual Review of Ecology and Systematics + 17:137–167. + + + + VITOUSEK + P + + + SANFORD + R + + + 1986 + Nutrient cycling in moist tropical forest + Annual Review of Ecology and Systematics + 17 + 137 + 167 + + + + WICKHAM, H. 2016. ggplot2: Elegant graphics for data + analysis. Springer. + + + + WICKHAM + H + + + 2016 + ggplot2: Elegant graphics for data analysis + Springer + + + WITTMANN, F., ANHUF, A. & JUNK, W.J. 2002. Tree species + distribution and community structure of central Amazonian várzea forests by + remote-sensing techniques. Journal of Tropical Ecology 18(6):805–820. + https://doi.org/10.1017/S0266467402002523 + + + + WITTMANN + F + + + ANHUF + A + + + JUNK + W.J + + + 2002 + Tree species distribution and community structure of central + Amazonian várzea forests by remote-sensing techniques + Journal of Tropical Ecology + 18 + 6 + 805 + 820 + 10.1017/S0266467402002523 + + + + WITTMANN, F. & JUNK, W.J. 2003. Sapling communities in Amazonian + white-water forests. Journal of Biogeography 30(10):1533–1544. + https://doi.org/10.1046/j.1365-2699.2003.00966.x + + + + WITTMANN + F + + + JUNK + W.J + + + 2003 + Sapling communities in Amazonian white-water + forests + Journal of Biogeography + 30 + 10 + 1533 + 1544 + 10.1046/j.1365-2699.2003.00966.x + + + + WITTMANN, F., SCHÖNGART, J., MONTEIRO, J.C., MOTZER, T., JUNK, W.J., + PIEDADE, M.T.F., QUEIROZ, H.L. & WORBES, M. 2006. Tree species composition + and diversity gradients in white-water forests across the Amazon Basin. Journal + of Biogeography 33(8):1334–1347. + https://doi.org/10.1111/j.1365-2699.2006.01495.x + + + + WITTMANN + F + + + SCHÖNGART + J + + + MONTEIRO + J.C + + + MOTZER + T + + + JUNK + W.J + + + PIEDADE + M.T.F + + + QUEIROZ + H.L + + + WORBES + M + + + 2006 + Tree species composition and diversity gradients in white-water + forests across the Amazon Basin + Journal of Biogeography + 33 + 8 + 1334 + 1347 + 10.1111/j.1365-2699.2006.01495.x + + + + WITTMANN, F., SCHÖNGART, J., BRITO, J.M., WITTMANN, A.O., PIEDADE, + M.T.F., PAROLIN, P., JUNK, W.J. & GUILLAUMET, J.J. 2010. Manual of + trees from Central Amazonian várzea floodplains: Taxonomy, ecology and + use. INPA. + + + + WITTMANN + F + + + SCHÖNGART + J + + + BRITO + J.M + + + WITTMANN + A.O + + + PIEDADE + M.T.F + + + PAROLIN + P + + + JUNK + W.J + + + GUILLAUMET + J.J + + + 2010 + Manual of trees from Central Amazonian várzea floodplains: + Taxonomy, ecology and use + INPA + + + + WITTMANN, F., ANHUF, D., JUNK, W.J. & PIEDADE, M.T.F. 2013. + Biogeography and biodiversity of Amazonian palm swamps. In W.J. Junk, M.T.F. + Piedade, F. Wittmann, J. Schöngart, & P. Parolin (Eds.), Amazonian + floodplain forests: Ecophysiology, biodiversity and sustainable + management (pp. 355–372). Springer. + + + + WITTMANN + F + + + ANHUF + D + + + JUNK + W.J + + + PIEDADE + M.T.F + + + 2013 + Biogeography and biodiversity of Amazonian palm + swamps + + + JUNK + W.J + + + PIEDADE + M.T.F + + + WITTMANN + F + + + SCHÖNGART + J + + + PAROLIN + P + + Eds + + Amazonian floodplain forests: Ecophysiology, biodiversity and + sustainable management (pp. 355–372) + Springer + + + Bahrami A, Hasanzadeh M, Hassanian SM, ShahidSales S, Ghayour-Mobarhan M, Ferns GA and Avan A (2017) The potential value of the PI3K/Akt/mTOR signaling pathway for assessing prognosis in cervical cancer and as a target for therapy. J Cell Biochem 118:4163–4169.BahramiAHasanzadehMHassanianSMShahidSalesSGhayour-MobarhanMFernsGAAvanA2017The potential value of the PI3K/Akt/mTOR signaling pathway for assessing prognosis in cervical cancer and as a target for therapyJ Cell Biochem11841634169Bosch FX, Lorincz A, Muñoz N, Meijer CJLM and Shah K V. (2002) The causal relation between human papillomavirus and cervical cancer. J Clin Pathol 55:244.BoschFXLorinczAMuñozNMeijerCJLMShahK V2002The causal relation between human papillomavirus and cervical cancerJ Clin Pathol55244244Nassu, M. P., Thyssen, P. J., Linhares, A. X. 2014. Developmental rate of immatures of two fly species of forensic importance: Sarcophaga (Liopygia) ruficornis and Microcerella halli (Diptera: Sarcophagidae). Parasitol. Res. 113(1), 217-222. https://doi.org/10.1007/s00436-013-3646-2NassuM. PThyssenP. JLinharesA. X2014Developmental rate of immatures of two fly species of forensic importance: Sarcophaga (Liopygia) ruficornis and Microcerella halli (Diptera: Sarcophagidae)Parasitol. Res.113121722210.1007/s00436-013-3646-2Newton, A. 2022. StaphBase (version Aug 2022). In: O. Bánki et al. (eds), Catalogue of Life (2025-10-10 XR). Amsterdam: Catalogue of Life Foundation, 2025.NewtonA2022StaphBase (version Aug 2022)Catalogue of Life FoundationCatalogue of Life (2025-10-10 XR)Pires, S. M., Cárcamo, M. C., Zimmer, C. R., Ribeiro, P. B. 2009. Influence of diet on the development and reproductive investment of Chrysomya megacephala (Fabricius, 1794) (Diptera: Calliphoridae). Arq. Inst. Biol. (Sao Paulo). 76, 41-47. https://doi.org/10.1590/1808-1657v76p0412009PiresS. MCárcamoM. CZimmerC. RRibeiroP. B2009Influence of diet on the development and reproductive investment of Chrysomya megacephala (Fabricius, 1794) (Diptera: Calliphoridae)Arq. Inst. Biol. (Sao Paulo)76414710.1590/1808-1657v76p0412009 diff --git a/reference/marking.py b/reference/marking.py new file mode 100644 index 0000000..f100b5d --- /dev/null +++ b/reference/marking.py @@ -0,0 +1,115 @@ +import json +import logging + +from django.conf import settings + +from reference.exceptions import ( + ReferenceLlamaDisabledError, + ReferenceLlamaMisconfiguredError, + ReferenceLlamaUnavailableError, +) +from reference.prompts import BATCH_RESPONSE_FORMAT, MESSAGES, RESPONSE_FORMAT +from reference.providers import get_provider +from reference.utils.references import parse_reference_list + +logger = logging.getLogger(__name__) + + +def mark_reference(reference_text): + try: + reference_marker = get_provider(MESSAGES, RESPONSE_FORMAT) + output = reference_marker.run(reference_text) + for item in output.get("choices", []): + yield item.get("message", {}).get("content", "") + + except ( + ReferenceLlamaDisabledError, + ReferenceLlamaMisconfiguredError, + ReferenceLlamaUnavailableError, + ) as exc: + logger.error( + "Error marking reference via Llama: %s — ref=%s", exc, reference_text + ) + raise + + except Exception as exc: + logger.exception("Unexpected error marking reference: ref=%s", reference_text) + yield f"An unexpected error occurred: {str(exc)}" + + +def mark_reference_texts(texts): + lines = list(texts) + if not lines: + return [] + + batch_size = max(1, int(getattr(settings, "REFERENCE_BATCH_SIZE", 10) or 10)) + marked = [] + for start in range(0, len(lines), batch_size): + chunk = lines[start : start + batch_size] + if len(chunk) == 1: + choices = list(mark_reference(chunk[0])) + marked.append(choices[0] if choices else None) + continue + + batch_contents = None + try: + reference_marker = get_provider(MESSAGES, BATCH_RESPONSE_FORMAT) + numbered = "\n".join( + f"{index}. {line}" for index, line in enumerate(chunk, start=1) + ) + user_input = ( + "Extract each numbered line. Respond ONLY with a JSON object " + '{"results":[...]} with exactly one object per line in the same ' + 'order. Use {"is_reference": false} for non-citations.\n\n' + f"{numbered}" + ) + output = reference_marker.run(user_input) + content = "" + for item in output.get("choices", []): + content = item.get("message", {}).get("content", "") + break + parsed = json.loads(content) if content else None + results = None + if isinstance(parsed, list): + results = parsed + elif isinstance(parsed, dict): + if isinstance(parsed.get("results"), list): + results = parsed["results"] + if results is not None and len(results) == len(chunk): + batch_contents = [ + json.dumps(item) if isinstance(item, dict) else item + for item in results + ] + except ( + ReferenceLlamaDisabledError, + ReferenceLlamaMisconfiguredError, + ReferenceLlamaUnavailableError, + ): + raise + except Exception: + logger.exception( + "Batch marking failed for %d references; falling back to one-by-one", + len(chunk), + ) + + if batch_contents is None: + logger.warning( + "Batch mark unavailable for %d refs; falling back to one-by-one", + len(chunk), + ) + for line in chunk: + choices = list(mark_reference(line)) + marked.append(choices[0] if choices else None) + else: + marked.extend(batch_contents) + + return marked + + +def mark_references(reference_block): + lines = parse_reference_list(reference_block) + for ref_row, content in zip(lines, mark_reference_texts(lines)): + yield { + "references": ref_row, + "choices": [content] if content is not None else [], + } diff --git a/reference/migrations/0001_initial.py b/reference/migrations/0001_initial.py new file mode 100644 index 0000000..da2d721 --- /dev/null +++ b/reference/migrations/0001_initial.py @@ -0,0 +1,148 @@ +# Generated by Django 6.0.5 on 2026-07-03 11:43 + +import django.core.validators +import django.db.models.deletion +import modelcluster.fields +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="Reference", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "created", + models.DateTimeField( + auto_now_add=True, verbose_name="Creation date" + ), + ), + ( + "updated", + models.DateTimeField( + auto_now=True, verbose_name="Last update date" + ), + ), + ( + "mixed_citation", + models.TextField(blank=True, verbose_name="Mixed Citation"), + ), + ( + "normalized_citation", + models.TextField( + blank=True, db_index=True, verbose_name="Normalized citation" + ), + ), + ( + "checksum", + models.CharField( + blank=True, max_length=64, unique=True, verbose_name="SHA256" + ), + ), + ( + "status", + models.IntegerField( + blank=True, + choices=[ + (0, "No reference"), + (1, "Creating reference"), + (2, "Reference ready"), + ], + default=0, + verbose_name="Reference status", + ), + ), + ( + "creator", + models.ForeignKey( + editable=False, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="%(class)s_creator", + to=settings.AUTH_USER_MODEL, + verbose_name="Creator", + ), + ), + ( + "updated_by", + models.ForeignKey( + blank=True, + editable=False, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="%(class)s_last_mod_user", + to=settings.AUTH_USER_MODEL, + verbose_name="Updater", + ), + ), + ], + options={ + "verbose_name": "Reference", + "verbose_name_plural": "References", + }, + ), + migrations.CreateModel( + name="ElementCitation", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "sort_order", + models.IntegerField(blank=True, editable=False, null=True), + ), + ( + "marked", + models.JSONField(blank=True, default=dict, verbose_name="Marked"), + ), + ("marked_xml", models.TextField(blank=True, verbose_name="Marked XML")), + ( + "score", + models.IntegerField( + blank=True, + help_text="Rating from 1 to 10", + null=True, + validators=[ + django.core.validators.MinValueValidator(1), + django.core.validators.MaxValueValidator(10), + ], + ), + ), + ( + "reference", + modelcluster.fields.ParentalKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="element_citation", + to="reference.reference", + ), + ), + ], + options={ + "ordering": ["sort_order"], + "abstract": False, + }, + ), + ] diff --git a/reference/migrations/0002_rename_app_referencia_to_reference.py b/reference/migrations/0002_rename_app_referencia_to_reference.py new file mode 100644 index 0000000..ab05151 --- /dev/null +++ b/reference/migrations/0002_rename_app_referencia_to_reference.py @@ -0,0 +1,64 @@ +from django.db import migrations + + +def _table_exists(schema_editor, name): + with schema_editor.connection.cursor() as cursor: + cursor.execute( + """ + SELECT 1 + FROM information_schema.tables + WHERE table_schema = current_schema() + AND table_name = %s + """, + [name], + ) + return cursor.fetchone() is not None + + +def rename_legacy_tables(apps, schema_editor): + mapping = ( + ("referencia_reference", "reference_reference"), + ("referencia_elementcitation", "reference_elementcitation"), + ) + with schema_editor.connection.cursor() as cursor: + for old_name, new_name in mapping: + old_exists = _table_exists(schema_editor, old_name) + new_exists = _table_exists(schema_editor, new_name) + if old_exists and not new_exists: + cursor.execute(f'ALTER TABLE "{old_name}" RENAME TO "{new_name}"') + elif old_exists and new_exists: + cursor.execute(f'DROP TABLE "{old_name}" CASCADE') + + +def revert_legacy_tables(apps, schema_editor): + mapping = ( + ("reference_reference", "referencia_reference"), + ("reference_elementcitation", "referencia_elementcitation"), + ) + with schema_editor.connection.cursor() as cursor: + for old_name, new_name in mapping: + old_exists = _table_exists(schema_editor, old_name) + new_exists = _table_exists(schema_editor, new_name) + if old_exists and not new_exists: + cursor.execute(f'ALTER TABLE "{old_name}" RENAME TO "{new_name}"') + + +def rename_app_label(apps, schema_editor): + ContentType = apps.get_model("contenttypes", "ContentType") + ContentType.objects.filter(app_label="referencia").update(app_label="reference") + + +def revert_app_label(apps, schema_editor): + ContentType = apps.get_model("contenttypes", "ContentType") + ContentType.objects.filter(app_label="reference").update(app_label="referencia") + + +class Migration(migrations.Migration): + dependencies = [ + ("reference", "0001_initial"), + ] + + operations = [ + migrations.RunPython(rename_legacy_tables, revert_legacy_tables), + migrations.RunPython(rename_app_label, revert_app_label), + ] diff --git a/reference/migrations/__init__.py b/reference/migrations/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/reference/migrations/__init__.py @@ -0,0 +1 @@ + diff --git a/reference/models.py b/reference/models.py new file mode 100644 index 0000000..d7bb56e --- /dev/null +++ b/reference/models.py @@ -0,0 +1,117 @@ +import hashlib + +from django.conf import settings +from django.core.validators import MaxValueValidator, MinValueValidator +from django.db import models +from django.utils.translation import gettext_lazy as _ +from modelcluster.fields import ParentalKey +from modelcluster.models import ClusterableModel +from wagtail.admin.forms import WagtailAdminModelForm +from wagtail.admin.panels import FieldPanel, InlinePanel +from wagtail.models import Orderable +from wagtail_json_widget.widgets import JSONEditorWidget + +from reference.utils.references import stz_norm + + +class ReferenceMetadataModel(models.Model): + created = models.DateTimeField(verbose_name=_("Creation date"), auto_now_add=True) + updated = models.DateTimeField(verbose_name=_("Last update date"), auto_now=True) + creator = models.ForeignKey( + settings.AUTH_USER_MODEL, + verbose_name=_("Creator"), + related_name="%(class)s_creator", + editable=False, + on_delete=models.SET_NULL, + null=True, + ) + updated_by = models.ForeignKey( + settings.AUTH_USER_MODEL, + verbose_name=_("Updater"), + related_name="%(class)s_last_mod_user", + editable=False, + null=True, + blank=True, + on_delete=models.SET_NULL, + ) + + class Meta: + abstract = True + + +class ReferenceStatus(models.IntegerChoices): + NO_REFERENCE = 0, _("No reference") + CREATING = 1, _("Creating reference") + READY = 2, _("Reference ready") + + +class Reference(ReferenceMetadataModel, ClusterableModel): + mixed_citation = models.TextField(_("Mixed Citation"), null=False, blank=True) + normalized_citation = models.TextField( + _("Normalized citation"), blank=True, db_index=True + ) + checksum = models.CharField(_("SHA256"), max_length=64, blank=True, unique=True) + + status = models.IntegerField( + _("Reference status"), + choices=ReferenceStatus.choices, + blank=True, + default=ReferenceStatus.NO_REFERENCE, + ) + + panels = [ + FieldPanel("mixed_citation"), + InlinePanel("element_citation", label=_("Cited Elements")), + ] + + base_form_class = WagtailAdminModelForm + + def __str__(self): + return self.mixed_citation + + def save(self, *args, **kwargs): + self.normalized_citation = stz_norm(self.mixed_citation) + self.checksum = hashlib.sha256( + self.normalized_citation.encode("utf-8") + ).hexdigest() + super().save(*args, **kwargs) + + class Meta: + verbose_name = _("Reference") + verbose_name_plural = _("References") + + +class ElementCitation(Orderable): + reference = ParentalKey( + Reference, + on_delete=models.SET_NULL, + related_name="element_citation", + null=True, + ) + marked = models.JSONField(_("Marked"), default=dict, blank=True) + marked_xml = models.TextField(_("Marked XML"), blank=True) + + score = models.IntegerField( + null=True, + blank=True, + validators=[ + MinValueValidator(1), + MaxValueValidator(10), + ], + help_text=_("Rating from 1 to 10"), + ) + + panels = [ + FieldPanel( + "marked", + widget=JSONEditorWidget( + options={ + "mode": "code", + "modes": ["code", "tree"], + "search": True, + } + ), + ), + FieldPanel("marked_xml"), + FieldPanel("score"), + ] diff --git a/reference/prompts.py b/reference/prompts.py new file mode 100644 index 0000000..cb5c512 --- /dev/null +++ b/reference/prompts.py @@ -0,0 +1,295 @@ +MESSAGES = [ + { + "role": "system", + "content": ( + "You extract bibliographic citation components and respond ONLY with " + "JSON. Single input → one JSON object. Numbered lines → " + '{"results":[...]} with exactly one object per line in order. ' + "Non-citations (figure/table captions, section headings, ORCID URLs, " + "standalone names, editorial roles such as SCIENTIFIC EDITOR, " + "CRediT/authorship statements like Responsibility for…) → " + '{"is_reference": false}. ' + "References always include reftype: book, confproc, data, database, " + "journal, legal-doc, letter, newspaper, patent, preprint, report, " + "software, thesis, webpage, other. " + "Person authors/editors: {surname, fname}; institution: {collab}. " + "Preserve surname/fname spelling and capitalization as in the citation; " + "keep initial clusters compact when the citation has no spaces (C.A not " + "C. A.) but do not invent missing initials. " + "date: string; keep letter suffixes (2013a, 2013b). " + "If the text has DOI: or doi.org/…, ALWAYS set doi to the bare id " + "(no https://doi.org/); do not emit uri when doi is present; never " + "invent doi or uri. At most one uri. " + "journal: title=article, source=periodical. Patterns like " + "Source, vol(num), pages or Source vol(num):pages or Source, vol, " + "pages → journal with vol, num when parentheses present, and " + "fpage/lpage (do not use pages for journal page ranges); pages only " + "for elocation-id. Volume+pages after a periodical name means " + "journal, not book. " + "book: whole work uses source only (do not use title); chapter uses " + "chapter + source; publisher → organization; thesis: source=title " + "(no title field). " + "data: title=dataset, source=repository. " + "R/CRAN packages and similar download pages → software (or webpage) " + "with uri when present. " + "Copy title/source wording verbatim (keep hyphens like Above-ground). " + "confproc/webpage/software/legal-doc: use fields when present." + ), + }, + { + "role": "user", + "content": ( + "Bachman, S., J. Moat, A. W. Hill, J. de la Torre and B. Scott. 2011. " + "Supporting Red List threat assessments with GeoCAT: geospatial " + "conservation assessment tool. ZooKeys 150: 117-126. DOI: " + "https://doi.org/10.3897/zookeys.150.2109" + ), + }, + { + "role": "assistant", + "content": ( + '{"reftype":"journal","authors":[{"surname":"Bachman","fname":"S."},' + '{"surname":"Moat","fname":"J."},{"surname":"Hill","fname":"A. W."},' + '{"surname":"de la Torre","fname":"J."},' + '{"surname":"Scott","fname":"B."}],"date":"2011",' + '"title":"Supporting Red List threat assessments with GeoCAT: ' + 'geospatial conservation assessment tool","source":"ZooKeys",' + '"vol":150,"fpage":"117","lpage":"126",' + '"doi":"10.3897/zookeys.150.2109"}' + ), + }, + { + "role": "user", + "content": ( + "Alvares, C. A., Stape, J. L., Sentelhas, P. C., Gonçalves, J. L. M., " + "& Sparovek, G. (2013b). Köppen’s climate classification map for " + "Brazil. Meteorologische Zeitschrift, 22(6), 711–728. " + "https://doi.org/10.1127/0941-2948/2013/0507" + ), + }, + { + "role": "assistant", + "content": ( + '{"reftype":"journal",' + '"authors":[{"surname":"Alvares","fname":"C. A."},' + '{"surname":"Stape","fname":"J. L."},' + '{"surname":"Sentelhas","fname":"P. C."},' + '{"surname":"Gonçalves","fname":"J. L. M."},' + '{"surname":"Sparovek","fname":"G."}],' + '"date":"2013b",' + '"title":"Köppen’s climate classification map for Brazil",' + '"source":"Meteorologische Zeitschrift",' + '"vol":22,"num":6,"fpage":"711","lpage":"728",' + '"doi":"10.1127/0941-2948/2013/0507"}' + ), + }, + { + "role": "user", + "content": ( + "Calkins BM, Mendeloff AI. The epidemiology of idiopathic inflammatory " + "bowel disease. In: Kirsner JB, Shorter RG, eds. Inflammatory bowel " + "disease, 4th ed. Baltimore: Williams & Wilkins. 1995:31-68." + ), + }, + { + "role": "assistant", + "content": ( + '{"reftype":"book","authors":[{"surname":"Calkins","fname":"BM"},' + '{"surname":"Mendeloff","fname":"AI"}],' + '"editors":[{"surname":"Kirsner","fname":"JB"},' + '{"surname":"Shorter","fname":"RG"}],"date":"1995",' + '"source":"Inflammatory bowel disease",' + '"chapter":"The epidemiology of idiopathic inflammatory bowel disease",' + '"edition":"4th","organization":"Williams & Wilkins",' + '"location":"Baltimore","fpage":"31","lpage":"68"}' + ), + }, + { + "role": "user", + "content": ( + "Anderson, J. M., & Ingram, J. S. I. (1993). Tropical soil biology and " + "fertility: A handbook of methods (2nd ed.). CAB International." + ), + }, + { + "role": "assistant", + "content": ( + '{"reftype":"book",' + '"authors":[{"surname":"Anderson","fname":"J. M."},' + '{"surname":"Ingram","fname":"J. S. I."}],' + '"date":"1993",' + '"source":"Tropical soil biology and fertility: A handbook of methods",' + '"edition":"2nd","organization":"CAB International"}' + ), + }, + { + "role": "user", + "content": ( + "Brunel, J. F. 1987. Sur le genre Phyllanthus L. Thèse de doctorat " + "de l’Université L. Pasteur. Strasbourg, France. 760 pp." + ), + }, + { + "role": "assistant", + "content": ( + '{"reftype":"thesis","authors":[{"surname":"Brunel","fname":"J. F."}],' + '"date":"1987","source":"Sur le genre Phyllanthus L.",' + '"degree":"doctorat","organization":"l’Université L. Pasteur",' + '"location":"Strasbourg, France","num_pages":760}' + ), + }, + { + "role": "user", + "content": ( + "Felix Ribeiro, K. A. (2025). Replication data for: Mauritia flexuosa. " + "SciELO Data. https://doi.org/10.48331/SCIELODATA.RIVAW4" + ), + }, + { + "role": "assistant", + "content": ( + '{"reftype":"data",' + '"authors":[{"surname":"Felix Ribeiro","fname":"K. A."}],' + '"date":"2025","title":"Replication data for: Mauritia flexuosa",' + '"source":"SciELO Data","doi":"10.48331/SCIELODATA.RIVAW4"}' + ), + }, + { + "role": "user", + "content": ( + "Augie, B. (2017). gridExtra: Miscellaneous functions for “Grid” " + "graphics (Version 2.3) [R package]. " + "https://CRAN.R-project.org/package=gridExtra" + ), + }, + { + "role": "assistant", + "content": ( + '{"reftype":"software","authors":[{"surname":"Augie","fname":"B."}],' + '"date":"2017","source":"gridExtra",' + '"version":"2.3",' + '"uri":"https://CRAN.R-project.org/package=gridExtra"}' + ), + }, + { + "role": "user", + "content": ( + "1. Bachman S et al. 2011. Supporting Red List. ZooKeys 150:117-126. " + "DOI: 10.3897/zookeys.150.2109\n" + "2. Figure 1. Map of the study area." + ), + }, + { + "role": "assistant", + "content": ( + '{"results":[' + '{"reftype":"journal","authors":[{"surname":"Bachman","fname":"S"}],' + '"date":"2011","title":"Supporting Red List","source":"ZooKeys",' + '"vol":150,"fpage":"117","lpage":"126",' + '"doi":"10.3897/zookeys.150.2109"},' + '{"is_reference": false}' + "]}" + ), + }, + { + "role": "user", + "content": "Figure 1. Map of the study area.", + }, + { + "role": "assistant", + "content": '{"is_reference": false}', + }, + { + "role": "user", + "content": "Figura 2. Densidade populacional na Amazônia.", + }, + { + "role": "assistant", + "content": '{"is_reference": false}', + }, + { + "role": "user", + "content": "https://orcid.org/0000-0003-4872-7252", + }, + { + "role": "assistant", + "content": '{"is_reference": false}', + }, + { + "role": "user", + "content": "SCIENTIFIC EDITOR", + }, + { + "role": "assistant", + "content": '{"is_reference": false}', + }, + { + "role": "user", + "content": ( + "Responsibility for all aspects of the content and the integrity of " + "the published article. Camila Lima Ribeiro, Marcelle Miranda da Silva." + ), + }, + { + "role": "assistant", + "content": '{"is_reference": false}', + }, +] + +ITEM_PROPERTIES = { + "is_reference": {"type": "boolean"}, + "reftype": {"type": "string"}, + "authors": {"type": "array", "items": {"type": "object"}}, + "editors": {"type": "array", "items": {"type": "object"}}, + "full_text": {"type": "string"}, + "date": {"type": "string"}, + "title": {"type": "string"}, + "source": {"type": "string"}, + "chapter": {"type": "string"}, + "chapter_title": {"type": "string"}, + "edition": {"type": "string"}, + "doi": {"type": "string"}, + "vol": {"type": "integer"}, + "num": {"type": "integer"}, + "pages": {"type": "string"}, + "fpage": {"type": "string"}, + "lpage": {"type": "string"}, + "uri": {"type": "string"}, + "organization": {"type": "string"}, + "location": {"type": "string"}, + "org_location": {"type": "string"}, + "num_pages": {"type": "integer"}, + "version": {"type": "string"}, + "access_id": {"type": "string"}, + "access_date": {"type": "string"}, + "degree": {"type": "string"}, + "conf_loc": {"type": "string"}, + "conf_date": {"type": "string"}, + "conf_num": {"type": "string"}, + "country": {"type": "string"}, +} + +RESPONSE_FORMAT = { + "type": "json_object", + "schema": { + "type": "object", + "properties": ITEM_PROPERTIES, + }, +} + +BATCH_RESPONSE_FORMAT = { + "type": "json_object", + "schema": { + "type": "object", + "properties": { + "results": { + "type": "array", + "items": { + "type": "object", + "properties": ITEM_PROPERTIES, + }, + }, + }, + "required": ["results"], + }, +} diff --git a/reference/providers/__init__.py b/reference/providers/__init__.py new file mode 100644 index 0000000..a25c443 --- /dev/null +++ b/reference/providers/__init__.py @@ -0,0 +1,5 @@ +from reference.providers.http import Provider + + +def get_provider(messages, response_format, **kwargs): + return Provider(messages, response_format, **kwargs) diff --git a/reference/providers/http.py b/reference/providers/http.py new file mode 100644 index 0000000..ff7241f --- /dev/null +++ b/reference/providers/http.py @@ -0,0 +1,103 @@ +import logging +import time + +import requests + +from reference.exceptions import ( + ReferenceLlamaDisabledError, + ReferenceLlamaMisconfiguredError, + ReferenceLlamaUnavailableError, +) + +logger = logging.getLogger(__name__) + + +class Provider: + def __init__( + self, + messages, + response_format, + temperature=0.0, + top_p=0.1, + max_tokens=4000, + ): + from django.conf import settings + + self.messages = messages or [] + self.response_format = response_format + self.temperature = temperature + self.top_p = top_p + self.max_tokens = max_tokens + + if not getattr(settings, "REFERENCE_ENABLED", True): + raise ReferenceLlamaDisabledError("Reference Llama is disabled.") + + self.url = (getattr(settings, "REFERENCE_URL", "") or "").rstrip("/") + if not self.url: + raise ReferenceLlamaMisconfiguredError( + "REFERENCE_URL is required when Reference Llama is enabled." + ) + + self.model = getattr(settings, "REFERENCE_MODEL", "") or "llama3.2:3b" + self.timeout = getattr(settings, "REFERENCE_TIMEOUT", 300) + self.token = getattr(settings, "REFERENCE_TOKEN", "") or "" + self.num_ctx = int(getattr(settings, "REFERENCE_NUM_CTX", 8192) or 8192) + + def run(self, user_input): + messages = self.messages.copy() + messages.append({"role": "user", "content": user_input}) + return self.chat(messages) + + def chat(self, messages): + started = time.monotonic() + logger.info( + "Reference Llama chat via %s model=%s. Preview: %r", + self.url, + self.model, + messages[-1]["content"][:150], + ) + + options = { + "temperature": self.temperature, + "top_p": self.top_p, + "num_ctx": self.num_ctx, + } + if self.max_tokens: + options["num_predict"] = self.max_tokens + + payload = { + "model": self.model, + "messages": messages, + "options": options, + "stream": False, + } + if self.response_format and self.response_format.get("type") == "json_object": + schema = self.response_format.get("schema") + payload["format"] = schema if schema else "json" + + headers = {} + if self.token: + headers["Authorization"] = f"Bearer {self.token}" + + try: + resp = requests.post( + f"{self.url}/api/chat", + json=payload, + headers=headers, + timeout=self.timeout, + ) + resp.raise_for_status() + response_text = resp.json().get("message", {}).get("content") or "" + except requests.RequestException as exc: + logger.error("Reference Llama HTTP error: %s", exc) + raise ReferenceLlamaUnavailableError( + f"Reference Llama service unavailable: {exc}" + ) from exc + + elapsed = time.monotonic() - started + logger.info( + "Reference Llama chat: %d chars in %.2fs", + len(response_text), + elapsed, + ) + return {"choices": [{"message": {"content": response_text}}]} diff --git a/reference/tests/__init__.py b/reference/tests/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/reference/tests/__init__.py @@ -0,0 +1 @@ + diff --git a/reference/tests/test_coverage.py b/reference/tests/test_coverage.py new file mode 100644 index 0000000..a217b1b --- /dev/null +++ b/reference/tests/test_coverage.py @@ -0,0 +1,698 @@ +import json +from unittest.mock import MagicMock + +import pytest +from django.contrib.auth import get_user_model +from django.core.files.uploadedfile import SimpleUploadedFile +from django.http import HttpResponseRedirect +from lxml import etree +from rest_framework.test import APIClient + +from reference.api.v1.serializers import ( + ReferenceDocxRequestSerializer, + ReferencesInputField, +) +from reference.api.v1.views import ReferenceViewSet +from reference.create_forms import ReferenceCreateAdminForm +from reference.data_utils import ( + append_access_date, + append_citation_pages, + build_ref_list, + get_number_of_month, + get_reference, + get_xml, + parse_marked_choice, + resolve_reference_result, + resolve_references_result, +) +from reference.marking import mark_reference +from reference.models import ElementCitation, Reference, ReferenceStatus +from reference.providers import get_provider +from reference.providers.http import Provider +from reference.tests.test_docx_api import make_docx_bytes +from reference.utils.references import extract_text_from_docx +from reference.wagtail_hooks import ReferenceCreateView + + +@pytest.mark.parametrize( + "texto,expected", + [ + ("cited may 2025", "05"), + ("cited 2025", None), + ], +) +def test_get_number_of_month(texto, expected): + assert get_number_of_month(texto) == expected + + +def test_append_citation_pages_empty(): + root = etree.Element("element-citation") + append_citation_pages(root, " ") + assert list(root) == [] + + +def test_append_access_date_with_and_without_year(): + with_year = etree.Element("element-citation") + append_access_date(with_year, "cited 2025") + node = with_year.find("date-in-citation") + assert node.get("iso-8601-date") == "2025-01-00" + + without_year = etree.Element("element-citation") + append_access_date(without_year, "cited yesterday") + node = without_year.find("date-in-citation") + assert node.get("iso-8601-date") is None + assert node.text == "cited yesterday" + + +def test_get_xml_malformed_json_returns_error(): + assert get_xml("{not-json").tag == "error" + + +def test_parse_marked_choice_dict_passthrough(): + marked = {"reftype": "journal", "title": "T"} + assert parse_marked_choice(marked) is marked + + +def test_get_xml_authors_collab_variants(): + xml_node = get_xml( + json.dumps( + { + "reftype": "journal", + "authors": [ + {"collab": "WHO"}, + {"surname": "Smith", "fname": "J", "collab": "Team X"}, + ], + "title": "T", + "source": "S", + "vol": 10, + "num": 2, + } + ) + ) + person_group = xml_node.find("person-group") + assert person_group.find("collab").text == "WHO" + name = person_group.find("name") + assert name.find("surname").text == "Smith" + assert name.find("collab").text == "Team X" + assert xml_node.find("volume").text == "10" + assert xml_node.find("issue").text == "2" + + +def test_get_xml_book_publisher_vol_doi(): + xml_node = get_xml( + json.dumps( + { + "reftype": "book", + "source": "Soil biology", + "vol": 2, + "publisher": "CAB", + "doi": "10.1000/book", + "date": 1993, + } + ) + ) + assert xml_node.find("volume").text == "2" + assert xml_node.find("publisher-name").text == "CAB" + assert xml_node.find("pub-id").text == "10.1000/book" + + +def test_get_xml_thesis(): + xml_node = get_xml( + json.dumps( + { + "reftype": "thesis", + "title": "PhD thesis title", + "degree": "PhD", + "organization": "USP", + "date": 2020, + } + ) + ) + assert xml_node.get("publication-type") == "thesis" + assert xml_node.find("source").text == "PhD thesis title" + assert xml_node.find("comment").text == "PhD" + assert xml_node.find("publisher-name").text == "USP" + + +def test_get_xml_confproc_location_num_and_org_location(): + xml_node = get_xml( + json.dumps( + { + "reftype": "confproc", + "title": "Proceedings of the 17th Workshop for Bishops", + "source": "Addiction and compulsive behaviors", + "location": "Dallas, TX", + "num": 17, + "organization": "National Catholic Bioethics Center (US)", + "org_location": "Boston", + "num_pages": 258, + "date": 2000, + } + ) + ) + assert xml_node.find("conf-loc").text == "Dallas, TX" + assert xml_node.find("conf-num").text == "17" + assert xml_node.find("publisher-loc").text == "Boston" + assert xml_node.find("size").text == "258" + + +def test_get_xml_data_access_id(): + xml_node = get_xml( + json.dumps( + { + "reftype": "data", + "title": "Dataset", + "source": "SciELO Data", + "doi": "https://doi.org/10.48331/scielodata.5Z4TMP", + "access_id": "UNF:6:Neyjad4du3rFprhupCXizA== [fileUNF]", + "date": 2024, + } + ) + ) + assert xml_node.find("pub-id").text == "10.48331/scielodata.5Z4TMP" + assert xml_node.find("comment").text.startswith("UNF:6:") + + +def test_get_xml_confproc(): + xml_node = get_xml( + json.dumps( + { + "reftype": "confproc", + "conf_name": "IGARSS", + "source": "Proceedings", + "conf_loc": "Toulouse", + "conf_date": 2003, + "conf_num": 23, + "organization": "IEEE", + "doi": "10.1000/conf", + } + ) + ) + assert xml_node.find("conf-name").text == "IGARSS" + assert xml_node.find("source").text == "Proceedings" + assert xml_node.find("conf-loc").text == "Toulouse" + assert xml_node.find("conf-date").text == "2003" + assert xml_node.find("conf-num").text == "23" + assert xml_node.find("publisher-name").text == "IEEE" + assert xml_node.find("pub-id").text == "10.1000/conf" + + titled = get_xml(json.dumps({"reftype": "confproc", "title": "Named Conference"})) + assert titled.find("conf-name").text == "Named Conference" + + +def test_get_xml_data_extra_fields(): + xml_node = get_xml( + json.dumps( + { + "reftype": "data", + "title": "Dataset", + "source": "SciELO Data", + "version": "1.0", + "uri": "https://example.org/data", + "organization": "SciELO", + "access_date": "cited 12 March 2025", + "date": 2025, + } + ) + ) + assert xml_node.find("version").text == "1.0" + assert xml_node.find("ext-link").text == "https://example.org/data" + assert xml_node.find("publisher-name").text == "SciELO" + assert xml_node.find("date-in-citation").get("iso-8601-date") == "2025-03-00" + + +def test_get_xml_webpage_software_legal_and_other(): + webpage = get_xml( + json.dumps( + { + "reftype": "webpage", + "source": "Home tips", + "country": "US", + "doi": "10.1000/web", + "access_date": "cited Jan 2024", + } + ) + ) + assert webpage.find("source").text == "Home tips" + assert webpage.find("publisher-loc").text == "US" + assert webpage.find("pub-id").text == "10.1000/web" + + software = get_xml( + json.dumps( + { + "reftype": "software", + "title": "R", + "version": "4.3", + "uri": "https://www.R-project.org/", + } + ) + ) + assert software.find("version").text == "4.3" + + legal = get_xml( + json.dumps( + { + "reftype": "legal-doc", + "title": "Lei 1/2020", + "organization": "Brasil", + } + ) + ) + assert legal.find("source").text == "Lei 1/2020" + + other = get_xml( + json.dumps( + { + "reftype": "other", + "source": "Misc source", + "doi": "10.1000/other", + "uri": "https://example.org/other", + } + ) + ) + assert other.find("source").text == "Misc source" + assert other.find("pub-id").text == "10.1000/other" + assert other.find("ext-link").text == "https://example.org/other" + + other_title = get_xml(json.dumps({"reftype": "other", "title": "Other title"})) + assert other_title.find("source").text == "Other title" + + +def test_build_ref_list_skips_missing_and_invalid_xml(): + xml_text = build_ref_list( + [ + {"mixed_citation": "Missing data"}, + {"mixed_citation": "Bad xml", "data": ""}, + ] + ) + root = etree.fromstring(xml_text.encode("utf-8")) + refs = root.findall("ref") + assert refs[0].find("element-citation") is None + assert refs[1].find("element-citation") is None + + +@pytest.mark.django_db +def test_get_reference_handles_invalid_json_choice(monkeypatch): + monkeypatch.setattr( + "reference.data_utils.mark_references", + lambda _block: iter([{"references": "Ref A", "choices": ["{not-json"]}]), + ) + reference = Reference.objects.create( + mixed_citation="Ref A", + status=ReferenceStatus.CREATING, + ) + get_reference(reference.id) + reference.refresh_from_db() + assert reference.status == ReferenceStatus.READY + marked = list(reference.element_citation.values_list("marked", flat=True)) + assert marked == [{"raw": "{not-json"}] + + +@pytest.mark.django_db +def test_get_reference_skips_choice_without_reftype(monkeypatch): + monkeypatch.setattr( + "reference.data_utils.mark_references", + lambda _block: iter( + [ + { + "references": "Ref A", + "choices": [ + {"title": "No type"}, + {"reftype": "journal", "title": "Ok"}, + ], + } + ] + ), + ) + reference = Reference.objects.create( + mixed_citation="Ref A", + status=ReferenceStatus.CREATING, + ) + get_reference(reference.id) + reference.refresh_from_db() + assert reference.status == ReferenceStatus.READY + marked = list(reference.element_citation.values_list("marked", flat=True)) + assert marked == [{"reftype": "journal", "title": "Ok"}] + + +@pytest.mark.django_db +def test_resolve_reference_result_ignores_mark_without_reftype(monkeypatch): + monkeypatch.setattr( + "reference.data_utils.mark_reference", + lambda _text: iter([{"title": "Missing type"}]), + ) + before_refs = Reference.objects.count() + result = resolve_reference_result("Incomplete mark citation.") + assert result is None + assert Reference.objects.count() == before_refs + + +@pytest.mark.django_db +def test_get_reference_reraises_missing_object(): + with pytest.raises(Reference.DoesNotExist): + get_reference(999999) + + +def test_marking_reports_unexpected_error(monkeypatch): + monkeypatch.setattr( + "reference.marking.get_provider", + lambda *a, **k: (_ for _ in ()).throw(RuntimeError("boom")), + ) + result = list(mark_reference("Ref A")) + assert len(result) == 1 + assert "unexpected error" in result[0].lower() + assert "boom" in result[0] + + +def test_get_provider_returns_provider(settings): + settings.REFERENCE_ENABLED = True + settings.REFERENCE_URL = "http://llama.example:11434" + provider = get_provider([], None) + assert isinstance(provider, Provider) + assert provider.url == "http://llama.example:11434" + + +@pytest.mark.django_db +def test_reference_str(): + reference = Reference.objects.create(mixed_citation="Smith J. Nature. 2024.") + assert str(reference) == "Smith J. Nature. 2024." + + +def test_references_input_field_to_representation(): + field = ReferencesInputField() + assert field.to_representation(["Ref A"]) == ["Ref A"] + + +def test_docx_serializer_rejects_empty_file(): + from rest_framework.exceptions import ValidationError + + upload = MagicMock() + upload.name = "empty.docx" + upload.size = 0 + serializer = ReferenceDocxRequestSerializer() + with pytest.raises(ValidationError, match="Empty file"): + serializer.validate_file(upload) + + +def test_api_reference_rejects_non_mapping_body(): + view = ReferenceViewSet() + request = MagicMock() + request.data = ["not", "a", "mapping"] + response = view.api_reference(request) + assert response.status_code == 400 + assert json.loads(response.content) == {"error": "Error processing"} + + +@pytest.mark.django_db +def test_api_docx_read_failure(monkeypatch): + monkeypatch.setattr( + "reference.utils.references.extract_text_from_docx", + lambda *a, **k: (_ for _ in ()).throw(OSError("broken")), + ) + User = get_user_model() + user = User.objects.create_user(username="docxfail", password="pass") + client = APIClient() + client.force_authenticate(user=user) + upload = SimpleUploadedFile( + "article.docx", + b"PK fake", + content_type=( + "application/vnd.openxmlformats-officedocument." "wordprocessingml.document" + ), + ) + response = client.post( + "/api/v1/reference/docx/", + data={"file": upload}, + format="multipart", + ) + assert response.status_code == 400 + assert response.json()["error"] == "Could not read DOCX file" + + +@pytest.mark.django_db +def test_reference_create_view_form_valid(monkeypatch): + User = get_user_model() + user = User.objects.create_user(username="wagtail-ref", password="pass") + resolve_calls = [] + + def fake_get_reference(obj_id): + reference = Reference.objects.get(id=obj_id) + ElementCitation.objects.create( + reference=reference, + marked={"reftype": "journal", "title": reference.mixed_citation}, + marked_xml="", + ) + reference.status = ReferenceStatus.READY + reference.save() + + monkeypatch.setattr("reference.data_utils.get_reference", fake_get_reference) + + original_resolve = resolve_references_result + + def tracking_resolve(references, user=None, output_type="json"): + resolve_calls.append( + {"references": references, "user": user, "output_type": output_type} + ) + return original_resolve(references, user=user, output_type=output_type) + + monkeypatch.setattr( + "reference.wagtail_hooks.resolve_references_result", + tracking_resolve, + ) + + view = ReferenceCreateView() + view.request = MagicMock(user=user) + view.get_success_url = lambda: "/admin/snippets/reference/reference/" + + citation_text = ( + "Smith J. Nature. 2024.\n\n" "Doe A. Science. 2023.\n" "Smith J. Nature. 2024." + ) + form = MagicMock() + form.cleaned_data = {"mixed_citation": citation_text} + + response = view.form_valid(form) + + assert isinstance(response, HttpResponseRedirect) + assert len(resolve_calls) == 1 + assert resolve_calls[0]["references"] == citation_text + assert resolve_calls[0]["user"] == user + assert resolve_calls[0]["output_type"] == "json" + assert Reference.objects.count() == 2 + assert all(ref.status == ReferenceStatus.READY for ref in Reference.objects.all()) + assert all(ref.element_citation.exists() for ref in Reference.objects.all()) + + +@pytest.mark.django_db +def test_reference_create_view_shows_error_when_llama_unavailable(monkeypatch): + from reference.exceptions import ReferenceLlamaUnavailableError + + User = get_user_model() + user = User.objects.create_user(username="wagtail-llama-down", password="pass") + error_messages = [] + + def raise_unavailable(*_args, **_kwargs): + raise ReferenceLlamaUnavailableError( + "Reference Llama service unavailable: 404 Client Error" + ) + + monkeypatch.setattr( + "reference.wagtail_hooks.resolve_references_result", + raise_unavailable, + ) + monkeypatch.setattr( + "reference.wagtail_hooks.messages.error", + lambda request, message: error_messages.append(str(message)), + ) + + view = ReferenceCreateView() + view.request = MagicMock(user=user) + view.render_to_response = MagicMock(return_value="rendered") + view.get_context_data = MagicMock(return_value={"form": MagicMock()}) + + form = MagicMock() + form.cleaned_data = {"mixed_citation": "Smith J. Nature. 2024."} + + before_refs = Reference.objects.count() + response = view.form_valid(form) + + assert response == "rendered" + assert Reference.objects.count() == before_refs + assert len(error_messages) == 1 + assert "Llama model is not available" in error_messages[0] + assert "404" in error_messages[0] + + +@pytest.mark.django_db +def test_reference_create_view_keeps_panels_and_docx(): + view = ReferenceCreateView() + view.model = Reference + view.panel = view.get_panel() + form_class = view.get_form_class() + + assert "mixed_citation" in form_class.base_fields + assert "docx_file" in form_class.base_fields + assert "element_citation" in form_class.formsets + panel_fields = [ + getattr(child, "field_name", None) or getattr(child, "relation_name", None) + for child in view.panel.children + ] + assert panel_fields == [ + "mixed_citation", + "docx_file", + "element_citation", + ] + + +@pytest.mark.django_db +def test_reference_create_admin_form_rejects_non_docx(): + form = ReferenceCreateAdminForm( + data={"mixed_citation": ""}, + files={ + "docx_file": SimpleUploadedFile( + "article.txt", + b"References\nRef A", + content_type="text/plain", + ) + }, + ) + assert not form.is_valid() + assert "docx_file" in form.errors + + +def test_reference_create_admin_form_clean_docx_rejects_empty_file(): + from django.core.exceptions import ValidationError + + class EmptyUpload: + name = "empty.docx" + size = 0 + + form = ReferenceCreateAdminForm() + form.cleaned_data = {"docx_file": EmptyUpload()} + with pytest.raises(ValidationError, match="Empty file"): + form.clean_docx_file() + + +def test_extract_text_from_docx_respects_limit_chars(tmp_path): + docx_path = tmp_path / "sample.docx" + docx_path.write_bytes( + make_docx_bytes(["References", "Smith J. Nature. 2024." * 20]) + ) + text = extract_text_from_docx(str(docx_path), limit_chars=40) + assert len(text) == 40 + + +@pytest.mark.django_db +def test_reference_create_admin_form_requires_text_or_docx(): + form = ReferenceCreateAdminForm(data={"mixed_citation": " "}) + assert not form.is_valid() + assert form.non_field_errors() + + +@pytest.mark.django_db +def test_reference_create_admin_form_rejects_docx_without_section(): + upload = SimpleUploadedFile( + "article.docx", + make_docx_bytes(["Introduction", "No refs here"]), + content_type=( + "application/vnd.openxmlformats-officedocument." "wordprocessingml.document" + ), + ) + form = ReferenceCreateAdminForm( + data={"mixed_citation": ""}, + files={"docx_file": upload}, + ) + assert not form.is_valid() + assert "No references section found in DOCX" in form.errors.as_text() + + +@pytest.mark.django_db +def test_reference_create_admin_form_docx_extracts_references(): + upload = SimpleUploadedFile( + "article.docx", + make_docx_bytes( + [ + "Introduction", + "References", + "Smith J. Nature. 2024.", + "Doe A. Science. 2023.", + ] + ), + content_type=( + "application/vnd.openxmlformats-officedocument." "wordprocessingml.document" + ), + ) + form = ReferenceCreateAdminForm( + data={"mixed_citation": "ignored text"}, + files={"docx_file": upload}, + ) + assert form.is_valid(), form.errors + assert form.cleaned_data["mixed_citation"] == ( + "Smith J. Nature. 2024.\nDoe A. Science. 2023." + ) + + +@pytest.mark.django_db +def test_reference_create_view_form_valid_from_docx(monkeypatch): + User = get_user_model() + user = User.objects.create_user(username="wagtail-docx", password="pass") + resolve_calls = [] + + def fake_get_reference(obj_id): + reference = Reference.objects.get(id=obj_id) + ElementCitation.objects.create( + reference=reference, + marked={"reftype": "journal", "title": reference.mixed_citation}, + marked_xml="", + ) + reference.status = ReferenceStatus.READY + reference.save() + + monkeypatch.setattr("reference.data_utils.get_reference", fake_get_reference) + + original_resolve = resolve_references_result + + def tracking_resolve(references, user=None, output_type="json"): + resolve_calls.append( + {"references": references, "user": user, "output_type": output_type} + ) + return original_resolve(references, user=user, output_type=output_type) + + monkeypatch.setattr( + "reference.wagtail_hooks.resolve_references_result", + tracking_resolve, + ) + + upload = SimpleUploadedFile( + "article.docx", + make_docx_bytes( + [ + "References", + "Smith J. Nature. 2024.", + "Doe A. Science. 2023.", + ] + ), + content_type=( + "application/vnd.openxmlformats-officedocument." "wordprocessingml.document" + ), + ) + form = ReferenceCreateAdminForm( + data={"mixed_citation": ""}, + files={"docx_file": upload}, + ) + assert form.is_valid(), form.errors + + view = ReferenceCreateView() + view.request = MagicMock(user=user) + view.get_success_url = lambda: "/admin/snippets/reference/reference/" + + response = view.form_valid(form) + + assert isinstance(response, HttpResponseRedirect) + assert len(resolve_calls) == 1 + assert resolve_calls[0]["references"] == ( + "Smith J. Nature. 2024.\nDoe A. Science. 2023." + ) + assert resolve_calls[0]["user"] == user + assert Reference.objects.count() == 2 + assert all(ref.status == ReferenceStatus.READY for ref in Reference.objects.all()) + assert all(ref.element_citation.exists() for ref in Reference.objects.all()) diff --git a/reference/tests/test_docx_api.py b/reference/tests/test_docx_api.py new file mode 100644 index 0000000..1257f20 --- /dev/null +++ b/reference/tests/test_docx_api.py @@ -0,0 +1,231 @@ +import io +import zipfile +from xml.sax.saxutils import escape + +import pytest +from django.contrib.auth import get_user_model +from django.core.files.uploadedfile import SimpleUploadedFile +from rest_framework.test import APIClient + +from reference.utils.references import extract_references_section + + +def make_docx_bytes(paragraphs): + body = "".join( + f"{escape(paragraph)}" + for paragraph in paragraphs + ) + document_xml = ( + '' + '' + f"{body}" + "" + ) + content_types = ( + '' + '' + '' + '' + "" + ) + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as archive: + archive.writestr("[Content_Types].xml", content_types) + archive.writestr("word/document.xml", document_xml) + return buffer.getvalue() + + +@pytest.mark.parametrize( + "text,expected", + [ + ("", ""), + ("Introduction\nMethods\nResults", ""), + ( + "Intro\nReferences\nSmith J. Nature. 2024.\nDoe A. Science. 2023.", + "Smith J. Nature. 2024.\nDoe A. Science. 2023.", + ), + ( + "Texto\n5. Referências\nRef A\nRef B", + "Ref A\nRef B", + ), + ( + "Texto\nBibliografia\nRef Unica", + "Ref Unica", + ), + ], +) +def test_extract_references_section(text, expected): + assert extract_references_section(text) == expected + + +@pytest.mark.django_db +def test_api_docx_marks_references(monkeypatch): + monkeypatch.setattr( + "reference.api.v1.views.resolve_references_result", + lambda references, user=None, output_type="json": [ + { + "mixed_citation": citation, + "data": {"reftype": "journal", "title": citation}, + } + for citation in references.split("\n") + if citation.strip() + ], + ) + + User = get_user_model() + user = User.objects.create_user(username="docxuser", password="pass") + client = APIClient() + client.force_authenticate(user=user) + + docx_bytes = make_docx_bytes( + [ + "Introduction", + "Some body text.", + "References", + "Smith J. Nature. 2024.", + "Doe A. Science. 2023.", + ] + ) + upload = SimpleUploadedFile( + "article.docx", + docx_bytes, + content_type=( + "application/vnd.openxmlformats-officedocument." "wordprocessingml.document" + ), + ) + + response = client.post( + "/api/v1/reference/docx/", + data={"file": upload, "type": "json"}, + format="multipart", + ) + + assert response.status_code == 200 + payload = response.json() + assert len(payload["references"]) == 2 + assert payload["references"][0]["mixed_citation"] == "Smith J. Nature. 2024." + assert payload["references"][1]["data"]["title"] == "Doe A. Science. 2023." + + +@pytest.mark.django_db +def test_api_docx_rejects_non_docx(): + User = get_user_model() + user = User.objects.create_user(username="docxbad", password="pass") + client = APIClient() + client.force_authenticate(user=user) + + upload = SimpleUploadedFile( + "article.txt", + b"References\nRef A", + content_type="text/plain", + ) + + response = client.post( + "/api/v1/reference/docx/", + data={"file": upload}, + format="multipart", + ) + + assert response.status_code == 400 + assert "file" in response.json() + + +@pytest.mark.django_db +def test_api_docx_rejects_missing_references_section(): + User = get_user_model() + user = User.objects.create_user(username="docxnoref", password="pass") + client = APIClient() + client.force_authenticate(user=user) + + upload = SimpleUploadedFile( + "article.docx", + make_docx_bytes(["Introduction", "No refs here"]), + content_type=( + "application/vnd.openxmlformats-officedocument." "wordprocessingml.document" + ), + ) + + response = client.post( + "/api/v1/reference/docx/", + data={"file": upload}, + format="multipart", + ) + + assert response.status_code == 400 + assert response.json()["error"] == "No references section found in DOCX" + + +@pytest.mark.django_db +def test_api_docx_requires_authentication(): + client = APIClient() + upload = SimpleUploadedFile( + "article.docx", + make_docx_bytes(["References", "Ref A"]), + content_type=( + "application/vnd.openxmlformats-officedocument." "wordprocessingml.document" + ), + ) + + response = client.post( + "/api/v1/reference/docx/", + data={"file": upload}, + format="multipart", + ) + + assert response.status_code in (401, 403) + + +@pytest.mark.django_db +def test_api_docx_get_renders_browsable_form(): + User = get_user_model() + user = User.objects.create_user(username="docxget", password="pass") + client = APIClient() + client.force_authenticate(user=user) + + response = client.get("/api/v1/reference/docx/") + + assert response.status_code == 200 + + +@pytest.mark.django_db +def test_api_docx_jats_returns_ref_list(monkeypatch): + monkeypatch.setattr( + "reference.api.v1.views.resolve_references_result", + lambda references, user=None, output_type="json": [ + { + "mixed_citation": "Smith J. Nature. 2024.", + "data": ( + '' + "Nature" + "" + ), + } + ], + ) + + User = get_user_model() + user = User.objects.create_user(username="docxjats", password="pass") + client = APIClient() + client.force_authenticate(user=user) + + upload = SimpleUploadedFile( + "article.docx", + make_docx_bytes(["Referências", "Smith J. Nature. 2024."]), + content_type=( + "application/vnd.openxmlformats-officedocument." "wordprocessingml.document" + ), + ) + + response = client.post( + "/api/v1/reference/docx/", + data={"file": upload, "type": "jats"}, + format="multipart", + ) + + assert response.status_code == 200 + payload = response.json() + assert "ref_list" in payload + assert "" in payload["ref_list"] diff --git a/reference/tests/test_http_provider.py b/reference/tests/test_http_provider.py new file mode 100644 index 0000000..d9dc65c --- /dev/null +++ b/reference/tests/test_http_provider.py @@ -0,0 +1,100 @@ +from unittest.mock import MagicMock, patch + +import pytest +import requests + +from reference.exceptions import ( + ReferenceLlamaDisabledError, + ReferenceLlamaMisconfiguredError, + ReferenceLlamaUnavailableError, +) +from reference.providers.http import Provider + + +@pytest.fixture +def llama_settings(settings): + settings.REFERENCE_ENABLED = True + settings.REFERENCE_URL = "http://llama.example:11434" + settings.REFERENCE_MODEL = "llama3.2:3b" + settings.REFERENCE_TIMEOUT = 30 + settings.REFERENCE_TOKEN = "" + settings.REFERENCE_NUM_CTX = 8192 + return settings + + +def test_http_provider_requires_url(settings): + settings.REFERENCE_ENABLED = True + settings.REFERENCE_URL = "" + + with pytest.raises(ReferenceLlamaMisconfiguredError): + Provider([], {"type": "json_object"}) + + +def test_http_provider_disabled(settings): + settings.REFERENCE_ENABLED = False + settings.REFERENCE_URL = "http://llama.example:11434" + + with pytest.raises(ReferenceLlamaDisabledError): + Provider([], {"type": "json_object"}) + + +def test_http_provider_chat_success(llama_settings): + mock_response = MagicMock() + mock_response.raise_for_status.return_value = None + mock_response.json.return_value = { + "message": {"content": '{"reftype":"journal"}'}, + } + + with patch( + "reference.providers.http.requests.post", return_value=mock_response + ) as post: + provider = Provider( + [{"role": "system", "content": "sys"}], + { + "type": "json_object", + "schema": { + "type": "object", + "properties": {"reftype": {"type": "string"}}, + "required": ["reftype"], + }, + }, + ) + result = provider.run("Smith J. Nature. 2024.") + + assert result == { + "choices": [{"message": {"content": '{"reftype":"journal"}'}}], + } + post.assert_called_once() + args, kwargs = post.call_args + assert args[0] == "http://llama.example:11434/api/chat" + assert kwargs["json"]["model"] == "llama3.2:3b" + assert kwargs["json"]["options"]["num_ctx"] == 8192 + assert kwargs["json"]["format"]["type"] == "object" + assert kwargs["json"]["format"]["required"] == ["reftype"] + assert kwargs["json"]["messages"][-1]["content"] == "Smith J. Nature. 2024." + assert kwargs["headers"] == {} + + +def test_http_provider_sends_bearer_token(llama_settings): + llama_settings.REFERENCE_TOKEN = "secret-token" + mock_response = MagicMock() + mock_response.raise_for_status.return_value = None + mock_response.json.return_value = {"message": {"content": "{}"}} + + with patch( + "reference.providers.http.requests.post", return_value=mock_response + ) as post: + provider = Provider([], None) + provider.chat([{"role": "user", "content": "hi"}]) + + assert post.call_args.kwargs["headers"]["Authorization"] == "Bearer secret-token" + + +def test_http_provider_raises_on_http_error(llama_settings): + with patch( + "reference.providers.http.requests.post", + side_effect=requests.ConnectionError("refused"), + ): + provider = Provider([], None) + with pytest.raises(ReferenceLlamaUnavailableError): + provider.chat([{"role": "user", "content": "hi"}]) diff --git a/reference/tests/test_marking.py b/reference/tests/test_marking.py new file mode 100644 index 0000000..56546a7 --- /dev/null +++ b/reference/tests/test_marking.py @@ -0,0 +1,636 @@ +import hashlib +import json + +import pytest + +from reference.data_utils import get_xml +from reference.exceptions import ReferenceLlamaMisconfiguredError +from reference.models import Reference +from reference.utils.references import stz_norm + + +class HttpProviderStub: + def __init__(self, *_args, **_kwargs): + pass + + def run(self, _reference_text): + return { + "choices": [ + {"message": {"content": '{"reftype":"journal","title":"Remote"}'}}, + ] + } + + +def test_marking_uses_http_llama(monkeypatch): + monkeypatch.setattr( + "reference.marking.get_provider", lambda *a, **k: HttpProviderStub() + ) + + from reference.marking import mark_reference + + result = list(mark_reference("Ref A")) + + assert result == ['{"reftype":"journal","title":"Remote"}'] + + +def test_mark_reference_texts_batches_one_request(monkeypatch, settings): + settings.REFERENCE_BATCH_SIZE = 10 + calls = [] + + class BatchProviderStub: + def __init__(self, messages, response_format, **_kwargs): + self.response_format = response_format + + def run(self, text): + calls.append(text) + return { + "choices": [ + { + "message": { + "content": json.dumps( + { + "results": [ + {"reftype": "journal", "title": "A"}, + {"reftype": "journal", "title": "B"}, + {"reftype": "journal", "title": "C"}, + ] + } + ) + } + } + ] + } + + monkeypatch.setattr( + "reference.marking.get_provider", + lambda *a, **k: BatchProviderStub(*a, **k), + ) + + from reference.marking import mark_reference_texts + + result = mark_reference_texts(["Ref A", "Ref B", "Ref C"]) + + assert len(calls) == 1 + assert "1. Ref A" in calls[0] + assert "2. Ref B" in calls[0] + assert "3. Ref C" in calls[0] + assert [json.loads(item)["title"] for item in result] == ["A", "B", "C"] + + +def test_mark_reference_texts_falls_back_on_count_mismatch(monkeypatch, settings): + settings.REFERENCE_BATCH_SIZE = 10 + calls = [] + + class MismatchThenSingleStub: + def __init__(self, messages, response_format, **_kwargs): + self.response_format = response_format + + def run(self, text): + calls.append(text) + schema = ( + self.response_format.get("schema", {}) if self.response_format else {} + ) + if isinstance(schema.get("properties"), dict) and "results" in schema.get( + "properties", {} + ): + return { + "choices": [ + { + "message": { + "content": json.dumps( + {"results": [{"reftype": "journal"}]} + ) + } + } + ] + } + return { + "choices": [ + { + "message": { + "content": json.dumps({"reftype": "journal", "title": text}) + } + } + ] + } + + monkeypatch.setattr( + "reference.marking.get_provider", + lambda *a, **k: MismatchThenSingleStub(*a, **k), + ) + + from reference.marking import mark_reference_texts + + result = mark_reference_texts(["Ref A", "Ref B"]) + + assert len(calls) == 3 + assert [json.loads(item)["title"] for item in result] == ["Ref A", "Ref B"] + + +def test_mark_reference_texts_single_uses_non_batch(monkeypatch, settings): + settings.REFERENCE_BATCH_SIZE = 10 + formats = [] + + class CaptureFormatStub: + def __init__(self, messages, response_format, **_kwargs): + formats.append(response_format) + + def run(self, _text): + return { + "choices": [ + {"message": {"content": '{"reftype":"journal","title":"One"}'}} + ] + } + + monkeypatch.setattr( + "reference.marking.get_provider", + lambda *a, **k: CaptureFormatStub(*a, **k), + ) + + from reference.marking import mark_reference_texts + from reference.prompts import RESPONSE_FORMAT + + result = mark_reference_texts(["Only one"]) + + assert len(formats) == 1 + assert formats[0] is RESPONSE_FORMAT + assert json.loads(result[0])["title"] == "One" + + +def test_marking_reports_llama_misconfigured(monkeypatch): + def raise_misconfigured(*_args, **_kwargs): + raise ReferenceLlamaMisconfiguredError("REFERENCE_URL is required.") + + monkeypatch.setattr("reference.marking.get_provider", raise_misconfigured) + + from reference.marking import mark_reference + + with pytest.raises(ReferenceLlamaMisconfiguredError, match="REFERENCE_URL"): + list(mark_reference("Ref A")) + + +def test_marking_raises_llama_unavailable(monkeypatch): + from reference.exceptions import ReferenceLlamaUnavailableError + + def raise_unavailable(*_args, **_kwargs): + raise ReferenceLlamaUnavailableError( + "Reference Llama service unavailable: 404 Client Error" + ) + + monkeypatch.setattr("reference.marking.get_provider", raise_unavailable) + + from reference.marking import mark_reference + + with pytest.raises(ReferenceLlamaUnavailableError, match="404"): + list(mark_reference("Ref A")) + + +def test_prompt_instructs_skip_for_figures(): + from reference.prompts import ( + BATCH_RESPONSE_FORMAT, + ITEM_PROPERTIES, + MESSAGES, + RESPONSE_FORMAT, + ) + + system = MESSAGES[0]["content"] + assert "is_reference" in system + assert "figure" in system.lower() or "Figure" in system + assert "orcid" in system.lower() + assert "SCIENTIFIC EDITOR" in system or "editorial" in system.lower() + assert "Responsibility" in system or "contribution" in system.lower() + assert "fpage" in system and "lpage" in system + assert "do not use pages for journal page ranges" in system + assert "whole work uses source only" in system + assert "bare id" in system.lower() or "without https://doi.org/" in system + assert "do not emit uri" in system + assert "2013a" in system or "letter suffix" in system.lower() + assert "vol(num)" in system or "parentheses" in system.lower() + assert ITEM_PROPERTIES["date"]["type"] == "string" + assert RESPONSE_FORMAT["schema"].get("required") is None + assert "results" in BATCH_RESPONSE_FORMAT["schema"]["properties"] + assert BATCH_RESPONSE_FORMAT["schema"]["required"] == ["results"] + for key in ( + "chapter", + "edition", + "fpage", + "lpage", + "location", + "org_location", + "num_pages", + "access_id", + "editors", + ): + assert key in ITEM_PROPERTIES + + pairs = list(zip(MESSAGES[1::2], MESSAGES[2::2])) + skip_examples = [ + user["content"] + for user, assistant in pairs + if assistant["content"] == '{"is_reference": false}' + ] + assert any("Figure 1" in text for text in skip_examples) + assert any("Figura" in text for text in skip_examples) + assert any("orcid.org" in text for text in skip_examples) + assert any("SCIENTIFIC EDITOR" in text for text in skip_examples) + assert any("Responsibility for" in text for text in skip_examples) + + journal_example = next( + assistant["content"] + for user, assistant in pairs + if '"reftype":"journal"' in assistant["content"] + and "2013b" in assistant["content"] + ) + assert '"date":"2013b"' in journal_example + assert '"num":6' in journal_example + assert '"doi":"10.1127/0941-2948/2013/0507"' in journal_example + assert '"fpage":"711"' in journal_example + assert '"lpage":"728"' in journal_example + assert "https://doi.org/" not in journal_example + + zoo_example = next( + assistant["content"] + for user, assistant in pairs + if "ZooKeys" in assistant["content"] and '"results"' not in assistant["content"] + ) + assert '"fpage":"117"' in zoo_example + assert '"lpage":"126"' in zoo_example + assert '"pages"' not in zoo_example + assert "https://doi.org/" not in zoo_example + + +def test_get_xml_journal(): + sample_json = json.dumps( + { + "reftype": "journal", + "authors": [{"surname": "Smith", "fname": "J"}], + "title": "Test Title", + "source": "Nature", + "date": "2024", + "doi": "10.1000/test", + } + ) + xml_node = get_xml(sample_json) + xml_text = json.dumps( + { + "tag": xml_node.tag, + "publication_type": xml_node.get("publication-type"), + "children": {child.tag: child.text for child in xml_node}, + } + ) + parsed = json.loads(xml_text) + + assert parsed["tag"] == "element-citation" + assert parsed["publication_type"] == "journal" + assert parsed["children"]["article-title"] == "Test Title" + assert parsed["children"]["source"] == "Nature" + assert parsed["children"]["year"] == "2024" + + person_group = xml_node.find("person-group") + name = person_group.find("name") + assert name.find("surname").text == "Smith" + assert name.find("given-names").text == "J" + pub_id = xml_node.find("pub-id") + assert pub_id.get("pub-id-type") == "doi" + assert pub_id.text == "10.1000/test" + + +def test_extract_doi_from_text_and_enrich(): + from reference.data_utils import ( + enrich_marked_from_citation, + extract_doi_from_text, + extract_vol_num_from_text, + ) + + citation = ( + "Alvares, C. A. (2013a). Modeling. Theoretical and Applied Climatology, " + "113, 407–427. https://doi.org/10.1007/s00704-012-0796-6" + ) + assert extract_doi_from_text(citation) == "10.1007/s00704-012-0796-6" + assert ( + extract_doi_from_text("DOI: 10.3897/zookeys.150.2109.") + == "10.3897/zookeys.150.2109" + ) + + enriched = enrich_marked_from_citation( + {"reftype": "journal", "title": "Modeling", "source": "TAC"}, + citation, + ) + assert enriched["doi"] == "10.1007/s00704-012-0796-6" + + from_uri = enrich_marked_from_citation( + { + "reftype": "journal", + "uri": "https://doi.org/10.1127/0941-2948/2013/0507", + }, + "No doi label here", + ) + assert from_uri["doi"] == "10.1127/0941-2948/2013/0507" + assert "uri" not in from_uri + + xml_node = get_xml(json.dumps(enriched)) + assert xml_node.find("pub-id[@pub-id-type='doi']").text == ( + "10.1007/s00704-012-0796-6" + ) + + issue_citation = ( + "Alvares, C. A. (2013b). Köppen’s climate classification map for Brazil. " + "Meteorologische Zeitschrift, 22(6), 711–728. " + "https://doi.org/10.1127/0941-2948/2013/0507" + ) + assert extract_vol_num_from_text(issue_citation) == { + "vol": 22, + "num": 6, + "fpage": "711", + "lpage": "728", + } + with_num = enrich_marked_from_citation( + {"reftype": "journal", "title": "Köppen", "source": "MZ"}, + issue_citation, + ) + assert with_num["vol"] == 22 + assert with_num["num"] == 6 + assert with_num["fpage"] == "711" + assert with_num["lpage"] == "728" + assert with_num["doi"] == "10.1127/0941-2948/2013/0507" + num_xml = get_xml(json.dumps(with_num)) + assert num_xml.find("volume").text == "22" + assert num_xml.find("issue").text == "6" + assert num_xml.find("fpage").text == "711" + assert num_xml.find("lpage").text == "728" + + from reference.data_utils import extract_uri_from_text + + cran = ( + "Augie, B. (2017). gridExtra: Miscellaneous functions for “Grid” graphics " + "(Version 2.3) [R package]. https://CRAN.R-project.org/package=gridExtra" + ) + assert extract_uri_from_text(cran) == "https://CRAN.R-project.org/package=gridExtra" + with_uri = enrich_marked_from_citation( + {"reftype": "software", "source": "gridExtra", "version": "2.3"}, + cran, + ) + assert with_uri["uri"] == "https://CRAN.R-project.org/package=gridExtra" + uri_xml = get_xml(json.dumps(with_uri)) + assert uri_xml.find("ext-link").text == ( + "https://CRAN.R-project.org/package=gridExtra" + ) + + web_doi = ( + "Brasil. (2024). Decreto. http://dx.doi.org/10.18542/ethnoscientia.v0i0.10245" + ) + web_enriched = enrich_marked_from_citation( + {"reftype": "webpage", "source": "Decreto"}, + web_doi, + ) + assert web_enriched["uri"] == "http://dx.doi.org/10.18542/ethnoscientia.v0i0.10245" + assert "doi" not in web_enriched or web_enriched.get("doi") in (None, "") + + +def test_get_xml_book(): + sample_json = json.dumps( + { + "reftype": "book", + "title": "Tropical soil biology", + "organization": "CAB International", + "date": 1993, + } + ) + xml_node = get_xml(sample_json) + assert xml_node.get("publication-type") == "book" + assert xml_node.find("source").text == "Tropical soil biology" + assert xml_node.find("publisher-name").text == "CAB International" + assert xml_node.find("year").text == "1993" + + +def test_get_xml_book_chapter_uses_part_title(): + xml_node = get_xml( + json.dumps( + { + "reftype": "book", + "chapter_title": "Mapping wetlands", + "source": "IGARSS proceedings", + "date": 2003, + "pages": "1375-1377", + } + ) + ) + assert xml_node.find("part-title").text == "Mapping wetlands" + assert xml_node.find("chapter-title") is None + assert xml_node.find("source").text == "IGARSS proceedings" + assert xml_node.find("fpage").text == "1375" + assert xml_node.find("lpage").text == "1377" + + +def test_get_xml_book_chapter_field_and_editors(): + xml_node = get_xml( + json.dumps( + { + "reftype": "book", + "chapter": "The epidemiology of idiopathic inflammatory bowel disease", + "source": "Inflammatory bowel disease", + "edition": "4th", + "editors": [{"surname": "Kirsner", "fname": "JB"}], + "organization": "Williams & Wilkins", + "location": "Baltimore", + "fpage": "31", + "lpage": "68", + "date": 1995, + } + ) + ) + assert xml_node.find("part-title").text.startswith("The epidemiology") + assert xml_node.find("edition").text == "4th" + assert xml_node.find("publisher-loc").text == "Baltimore" + assert xml_node.find("fpage").text == "31" + assert xml_node.find("lpage").text == "68" + editors = xml_node.find('person-group[@person-group-type="editor"]') + assert editors.find("name/surname").text == "Kirsner" + + +def test_get_xml_journal_pages_and_elocation(): + ranged = get_xml( + json.dumps( + { + "reftype": "journal", + "title": "A", + "source": "B", + "pages": "117-126", + } + ) + ) + assert ranged.find("fpage").text == "117" + assert ranged.find("lpage").text == "126" + + explicit = get_xml( + json.dumps( + { + "reftype": "journal", + "title": "A", + "source": "B", + "fpage": "117", + "lpage": "126", + "doi": "https://doi.org/10.3897/zookeys.150.2109", + } + ) + ) + assert explicit.find("fpage").text == "117" + assert explicit.find("lpage").text == "126" + assert explicit.find("pub-id").text == "10.3897/zookeys.150.2109" + + single = get_xml( + json.dumps( + { + "reftype": "journal", + "title": "A", + "source": "B", + "pages": "244", + } + ) + ) + assert single.find("fpage").text == "244" + assert single.find("lpage").text == "244" + + elocation = get_xml( + json.dumps( + { + "reftype": "journal", + "title": "A", + "source": "B", + "pages": "e240058", + } + ) + ) + assert elocation.find("elocation-id").text == "e240058" + assert elocation.find("fpage") is None + + +def test_get_xml_thesis_source_location_num_pages(): + xml_node = get_xml( + json.dumps( + { + "reftype": "thesis", + "source": "Sur le genre Phyllanthus L.", + "degree": "doctorat", + "organization": "l’Université L. Pasteur", + "location": "Strasbourg, France", + "num_pages": 760, + "date": 1987, + } + ) + ) + assert xml_node.find("source").text == "Sur le genre Phyllanthus L." + assert xml_node.find("publisher-loc").text == "Strasbourg, France" + size = xml_node.find("size") + assert size.get("units") == "pages" + assert size.text == "760" + + +def test_get_xml_data_uses_data_title(): + xml_node = get_xml( + json.dumps( + { + "reftype": "data", + "title": "Replication data for X", + "source": "SciELO Data", + "doi": "10.48331/scielodata.abc", + "date": 2025, + } + ) + ) + assert xml_node.get("publication-type") == "data" + assert xml_node.find("data-title").text == "Replication data for X" + assert xml_node.find("source").text == "SciELO Data" + + +def test_get_xml_software(): + xml_node = get_xml( + json.dumps( + { + "reftype": "software", + "title": "R: A language and environment", + "organization": "R Foundation", + "uri": "https://www.R-project.org/", + "date": 2021, + } + ) + ) + assert xml_node.get("publication-type") == "software" + assert xml_node.find("source").text == "R: A language and environment" + assert xml_node.find("ext-link").text == "https://www.R-project.org/" + + +def test_get_xml_missing_reftype_returns_error(): + xml_node = get_xml(json.dumps({"title": "No type"})) + assert xml_node.tag == "error" + + +def test_build_ref_list(): + from lxml import etree + + from reference.data_utils import build_ref_list + + results = [ + { + "mixed_citation": "Smith J. Nature. 2024.", + "data": ( + '' + "Nature paper" + '10.1/abc' + "" + ), + }, + { + "mixed_citation": "Doe A. Book title. Publisher.", + "data": ( + '' + "Book title" + "" + ), + }, + ] + xml_text = build_ref_list(results) + root = etree.fromstring(xml_text.encode("utf-8")) + + assert root.tag == "ref-list" + assert root.find("title").text == "References" + refs = root.findall("ref") + assert len(refs) == 2 + assert refs[0].get("id") == "B1" + assert refs[1].get("id") == "B2" + assert refs[0].find("mixed-citation").text == "Smith J. Nature. 2024." + assert refs[0].find("element-citation").get("publication-type") == "journal" + assert refs[0].find("element-citation/pub-id").text == "10.1/abc" + assert refs[1].find("element-citation").get("publication-type") == "book" + + +def test_build_ref_list_skips_error_element_citation(): + from lxml import etree + + from reference.data_utils import build_ref_list + + xml_text = build_ref_list( + [ + { + "mixed_citation": "Broken ref", + "data": "", + } + ] + ) + root = etree.fromstring(xml_text.encode("utf-8")) + ref = root.find("ref") + assert ref.find("mixed-citation").text == "Broken ref" + assert ref.find("element-citation") is None + + +@pytest.mark.django_db +def test_stz_norm_checksum(): + citation = "Smith J. Nature. 2024." + reference = Reference(mixed_citation=citation) + reference.save() + + expected_normalized = stz_norm(citation) + expected_checksum = hashlib.sha256(expected_normalized.encode("utf-8")).hexdigest() + + assert reference.normalized_citation == expected_normalized + assert reference.checksum == expected_checksum diff --git a/reference/tests/test_references.py b/reference/tests/test_references.py new file mode 100644 index 0000000..71fefa3 --- /dev/null +++ b/reference/tests/test_references.py @@ -0,0 +1,210 @@ +import json +import re + +import pytest +from lxml import etree + +from reference.data_utils import build_ref_list, get_xml +from reference.fixtures.references import REF_LIST_XML, REFERENCES +from reference.marking import mark_reference + + +def _collapse_ws(value): + if value is None: + return None + return re.sub(r"\s+", " ", str(value)).strip() + + +def _golden_element_citations(): + root = etree.fromstring(REF_LIST_XML.encode("utf-8")) + return [ref.find("element-citation") for ref in root.findall("ref")] + + +def _fields_from_element_citation(node): + if node is None: + return {} + fields = { + "reftype": node.get("publication-type"), + "date": _collapse_ws(node.findtext("year")), + "doi": _collapse_ws(node.findtext("pub-id[@pub-id-type='doi']")), + "vol": _collapse_ws(node.findtext("volume")), + "num": _collapse_ws(node.findtext("issue")), + "title": _collapse_ws( + node.findtext("article-title") + or node.findtext("part-title") + or node.findtext("data-title") + or node.findtext("conf-name") + or node.findtext("source") + ), + "source": _collapse_ws(node.findtext("source")), + } + fpage = _collapse_ws(node.findtext("fpage")) + lpage = _collapse_ws(node.findtext("lpage")) + elocation = _collapse_ws(node.findtext("elocation-id")) + if fpage and lpage: + fields["pages"] = f"{fpage}-{lpage}" + elif fpage: + fields["pages"] = fpage + elif elocation: + fields["pages"] = elocation + else: + fields["pages"] = None + uri_nodes = node.xpath(".//ext-link") + fields["uri"] = _collapse_ws(uri_nodes[0].text) if uri_nodes else None + authors = [] + person_group = node.find("person-group[@person-group-type='author']") + if person_group is not None: + for child in person_group: + if child.tag == "collab": + authors.append({"collab": _collapse_ws(child.text)}) + elif child.tag == "name": + authors.append( + { + "surname": _collapse_ws(child.findtext("surname")), + "fname": _collapse_ws(child.findtext("given-names")), + } + ) + fields["authors"] = authors + return fields + + +def _fields_from_marked_json(data): + pages = data.get("pages") + if pages is not None: + pages = _collapse_ws(str(pages).replace("–", "-").replace("—", "-")) + authors = [] + for author in data.get("authors") or []: + if ( + author.get("collab") + and not author.get("surname") + and not author.get("fname") + ): + authors.append({"collab": _collapse_ws(author.get("collab"))}) + else: + authors.append( + { + "surname": _collapse_ws(author.get("surname")), + "fname": _collapse_ws(author.get("fname")), + } + ) + title = data.get("title") or data.get("chapter_title") or data.get("source") + return { + "reftype": data.get("reftype"), + "date": _collapse_ws(data.get("date")), + "doi": _collapse_ws(data.get("doi")), + "vol": _collapse_ws(data.get("vol")), + "num": _collapse_ws(data.get("num")), + "pages": pages, + "title": _collapse_ws(title), + "source": _collapse_ws(data.get("source")), + "uri": _collapse_ws(data.get("uri")), + "authors": authors, + } + + +def _comparable_keys(expected_fields): + keys = ["reftype", "date", "doi", "vol", "num", "pages", "title", "source", "uri"] + return [key for key in keys if expected_fields.get(key) not in (None, "")] + + +def _assert_marked_matches_golden(marked, golden_node, ref_id): + expected = _fields_from_element_citation(golden_node) + actual = _fields_from_marked_json(marked) + for key in _comparable_keys(expected): + assert actual.get(key) == expected.get( + key + ), f"{ref_id} field {key}: expected {expected.get(key)!r}, got {actual.get(key)!r}" + if expected["authors"]: + assert len(actual["authors"]) >= min(3, len(expected["authors"])), ( + f"{ref_id} authors count: expected at least " + f"{min(3, len(expected['authors']))}, got {len(actual['authors'])}" + ) + for index, expected_author in enumerate( + expected["authors"][: min(3, len(expected["authors"]))] + ): + actual_author = actual["authors"][index] + for field_name, expected_value in expected_author.items(): + if expected_value in (None, ""): + continue + assert _collapse_ws(actual_author.get(field_name)) == expected_value, ( + f"{ref_id} author[{index}].{field_name}: " + f"expected {expected_value!r}, got {actual_author.get(field_name)!r}" + ) + + +def _assert_element_citation_matches(actual_node, golden_node, ref_id): + expected = _fields_from_element_citation(golden_node) + actual = _fields_from_element_citation(actual_node) + for key in _comparable_keys(expected): + assert actual.get(key) == expected.get(key), ( + f"{ref_id} JATS field {key}: expected {expected.get(key)!r}, " + f"got {actual.get(key)!r}" + ) + + +def _mark_eval_reference(index): + ref_id = f"B{index + 1}" + citation = REFERENCES[index] + choices = list(mark_reference(citation)) + assert choices, f"{ref_id}: Llama returned no choices" + raw = choices[0] + assert "Llama model is not available" not in raw, f"{ref_id}: {raw}" + assert "unexpected error" not in raw.lower(), f"{ref_id}: {raw}" + try: + marked = json.loads(raw) + except json.JSONDecodeError as exc: + raise AssertionError(f"{ref_id}: invalid JSON from Llama: {raw!r}") from exc + assert marked.get("reftype"), f"{ref_id}: missing reftype in {marked!r}" + return marked, raw + + +@pytest.fixture(scope="module") +def eval_llama_marked(): + return [_mark_eval_reference(index) for index in range(len(REFERENCES))] + + +def test_eval_corpus_aligned(): + golden_nodes = _golden_element_citations() + assert len(REFERENCES) == len(golden_nodes) + assert len(REFERENCES) == 103 + assert all(node is not None for node in golden_nodes) + + +@pytest.mark.llama +def test_eval_llama_json_matches_golden(eval_llama_marked): + golden_nodes = _golden_element_citations() + for index, golden_node in enumerate(golden_nodes): + marked, _raw = eval_llama_marked[index] + _assert_marked_matches_golden(marked, golden_node, f"B{index + 1}") + + +@pytest.mark.llama +def test_eval_llama_jats_matches_golden(eval_llama_marked): + golden_nodes = _golden_element_citations() + results = [] + for index, golden_node in enumerate(golden_nodes): + marked, raw = eval_llama_marked[index] + xml_node = get_xml(raw) + assert ( + xml_node.tag != "error" + ), f"B{index + 1}: get_xml returned error for {marked!r}" + _assert_element_citation_matches(xml_node, golden_node, f"B{index + 1}") + results.append( + { + "mixed_citation": REFERENCES[index], + "data": etree.tostring(xml_node, encoding="unicode"), + } + ) + + built = build_ref_list(results) + built_root = etree.fromstring( + built.replace( + "", + '', + 1, + ).encode("utf-8") + ) + built_nodes = [ref.find("element-citation") for ref in built_root.findall("ref")] + assert len(built_nodes) == len(golden_nodes) + for index, (built_node, golden_node) in enumerate(zip(built_nodes, golden_nodes)): + _assert_element_citation_matches(built_node, golden_node, f"B{index + 1}") diff --git a/reference/tests/test_references_input.py b/reference/tests/test_references_input.py new file mode 100644 index 0000000..32473ba --- /dev/null +++ b/reference/tests/test_references_input.py @@ -0,0 +1,843 @@ +import json + +import pytest +from django.contrib.auth import get_user_model +from django.db import IntegrityError +from rest_framework.test import APIClient + +from reference.data_utils import ( + get_reference, + resolve_reference_result, + resolve_references_result, +) +from reference.marking import mark_references +from reference.models import ElementCitation, Reference, ReferenceStatus +from reference.utils.references import parse_reference_list + + +@pytest.mark.parametrize( + "value,expected", + [ + (None, []), + ("", []), + ([], []), + (["", " ", "Ref A"], ["Ref A"]), + ("Ref A\n\nRef B", ["Ref A", "Ref B"]), + (["Ref A", "Ref B"], ["Ref A", "Ref B"]), + (("Ref A", "Ref B"), ["Ref A", "Ref B"]), + (123, ["123"]), + ], +) +def test_parse_reference_list(value, expected): + assert parse_reference_list(value) == expected + + +def test_mark_references_accepts_list(monkeypatch): + class BatchStub: + def __init__(self, messages, response_format, **_kwargs): + self.response_format = response_format + + def run(self, text): + schema = ( + self.response_format.get("schema", {}) if self.response_format else {} + ) + if isinstance(schema.get("properties"), dict) and "results" in schema.get( + "properties", {} + ): + return { + "choices": [ + { + "message": { + "content": json.dumps( + { + "results": [ + {"reftype": "journal", "title": "Ref A"}, + {"reftype": "journal", "title": "Ref B"}, + ] + } + ) + } + } + ] + } + return { + "choices": [ + {"message": {"content": '{"reftype":"journal"}'}}, + ] + } + + monkeypatch.setattr( + "reference.marking.get_provider", lambda *a, **k: BatchStub(*a, **k) + ) + + result = list(mark_references(["Ref A", "Ref B"])) + + assert len(result) == 2 + assert result[0]["references"] == "Ref A" + assert result[1]["references"] == "Ref B" + assert json.loads(result[0]["choices"][0])["title"] == "Ref A" + assert json.loads(result[1]["choices"][0])["title"] == "Ref B" + + +def test_mark_references_string_and_list_are_equivalent(monkeypatch): + class MarkStub: + def __init__(self, messages, response_format, **_kwargs): + self.response_format = response_format + + def run(self, text): + schema = ( + self.response_format.get("schema", {}) if self.response_format else {} + ) + if isinstance(schema.get("properties"), dict) and "results" in schema.get( + "properties", {} + ): + lines = [ + line.split(". ", 1)[1] + for line in text.splitlines() + if line[:1].isdigit() and ". " in line + ] + return { + "choices": [ + { + "message": { + "content": json.dumps( + { + "results": [ + {"reftype": "journal", "title": line} + for line in lines + ] + } + ) + } + } + ] + } + return { + "choices": [ + { + "message": { + "content": json.dumps({"reftype": "journal", "title": text}) + } + }, + ] + } + + monkeypatch.setattr( + "reference.marking.get_provider", lambda *a, **k: MarkStub(*a, **k) + ) + + from_list = list(mark_references(["Ref A", "Ref B"])) + from_string = list(mark_references("Ref A\nRef B")) + + assert [item["references"] for item in from_list] == [ + item["references"] for item in from_string + ] + assert from_list[0]["choices"] == from_string[0]["choices"] + + +@pytest.mark.django_db +def test_resolve_reference_result_enriches_cached_doi(monkeypatch): + citation = ( + "Alvares C. A. (2013a). Modeling monthly mean air temperature for Brazil. " + "Theoretical and Applied Climatology, 113, 407–427. " + "https://doi.org/10.1007/s00704-012-0796-6" + ) + reference = Reference.objects.create( + mixed_citation=citation, + status=ReferenceStatus.READY, + ) + ElementCitation.objects.create( + reference=reference, + marked={"reftype": "journal", "title": "Modeling", "source": "TAC"}, + marked_xml="", + ) + + result = resolve_reference_result(citation, output_type="json") + + assert result["data"]["doi"] == "10.1007/s00704-012-0796-6" + stored = reference.element_citation.first() + assert stored.marked["doi"] == "10.1007/s00704-012-0796-6" + assert "10.1007/s00704-012-0796-6" in stored.marked_xml + + +@pytest.mark.django_db +def test_resolve_reference_result_enriches_cached_num(): + citation = ( + "Alvares, C. A. (2013b). Köppen’s climate classification map for Brazil. " + "Meteorologische Zeitschrift, 22(6), 711–728." + ) + reference = Reference.objects.create( + mixed_citation=citation, + status=ReferenceStatus.READY, + ) + ElementCitation.objects.create( + reference=reference, + marked={"reftype": "journal", "title": "Köppen", "source": "MZ"}, + marked_xml="", + ) + + result = resolve_reference_result(citation, output_type="json") + + assert result["data"]["num"] == 6 + assert result["data"]["vol"] == 22 + stored = reference.element_citation.first() + assert stored.marked["num"] == 6 + assert "6" in stored.marked_xml + assert "22" in stored.marked_xml + + +@pytest.mark.django_db +def test_resolve_reference_result_reuses_existing_reference(): + reference = Reference.objects.create( + mixed_citation="Smith J. Nature. 2024.", + status=ReferenceStatus.READY, + ) + ElementCitation.objects.create( + reference=reference, + marked={"reftype": "journal", "title": "Cached"}, + marked_xml="", + ) + + result = resolve_reference_result("Smith J. Nature. 2024.", output_type="json") + + assert result["mixed_citation"] == "Smith J. Nature. 2024." + assert result["data"] == {"reftype": "journal", "title": "Cached"} + assert ( + Reference.objects.filter(mixed_citation="Smith J. Nature. 2024.").count() == 1 + ) + + +@pytest.mark.django_db +def test_resolve_reference_result_creates_and_marks_new_reference(monkeypatch): + monkeypatch.setattr( + "reference.data_utils.mark_reference", + lambda _text: iter([json.dumps({"reftype": "journal", "title": "Created"})]), + ) + + User = get_user_model() + user = User.objects.create_user(username="creator", password="pass") + + result = resolve_reference_result( + "Jones A. Science. 2023.", + user=user, + output_type="json", + ) + + assert result["data"] == {"reftype": "journal", "title": "Created"} + stored = Reference.objects.get(mixed_citation="Jones A. Science. 2023.") + assert stored.creator == user + assert stored.status == ReferenceStatus.READY + assert stored.element_citation.count() == 1 + + +@pytest.mark.django_db +def test_resolve_reference_result_handles_checksum_race(monkeypatch): + citation = "Tuffi Santos LD. Planta Daninha 2007; 25(1):133-37." + monkeypatch.setattr( + "reference.data_utils.mark_reference", + lambda _text: iter( + [json.dumps({"reftype": "journal", "title": "Crescimento do eucalipto"})] + ), + ) + + def racing_get_or_create(*args, **kwargs): + Reference.objects.create( + mixed_citation=citation, + status=ReferenceStatus.READY, + ) + raise IntegrityError("duplicate key value violates unique constraint") + + monkeypatch.setattr( + Reference.objects, + "get_or_create", + racing_get_or_create, + ) + + result = resolve_reference_result(citation, output_type="json") + + assert result is not None + assert result["mixed_citation"] == citation + assert Reference.objects.filter(mixed_citation=citation).count() == 1 + assert ( + ElementCitation.objects.filter(reference__mixed_citation=citation).count() == 1 + ) + + +@pytest.mark.django_db +def test_resolve_reference_result_ignores_figure_without_db(monkeypatch): + monkeypatch.setattr( + "reference.data_utils.mark_reference", + lambda _text: iter([json.dumps({"is_reference": False})]), + ) + + before_refs = Reference.objects.count() + before_cites = ElementCitation.objects.count() + + result = resolve_reference_result("Figure 1. Map of the study area.") + + assert result is None + assert Reference.objects.count() == before_refs + assert ElementCitation.objects.count() == before_cites + + +@pytest.mark.django_db +def test_resolve_reference_result_ignores_orcid_without_db(monkeypatch): + monkeypatch.setattr( + "reference.data_utils.mark_reference", + lambda _text: iter([json.dumps({"is_reference": False})]), + ) + + before_refs = Reference.objects.count() + before_cites = ElementCitation.objects.count() + + result = resolve_reference_result("https://orcid.org/0000-0003-4872-7252") + + assert result is None + assert Reference.objects.count() == before_refs + assert ElementCitation.objects.count() == before_cites + + +@pytest.mark.django_db +def test_resolve_reference_result_raises_without_db_when_llama_unavailable( + monkeypatch, +): + from reference.exceptions import ReferenceLlamaUnavailableError + + def raise_unavailable(_text): + raise ReferenceLlamaUnavailableError( + "Reference Llama service unavailable: 404 Client Error" + ) + yield # pragma: no cover + + monkeypatch.setattr("reference.data_utils.mark_reference", raise_unavailable) + + before_refs = Reference.objects.count() + before_cites = ElementCitation.objects.count() + + with pytest.raises(ReferenceLlamaUnavailableError, match="404"): + resolve_reference_result("Smith J. Nature. 2024.") + + assert Reference.objects.count() == before_refs + assert ElementCitation.objects.count() == before_cites + + +@pytest.mark.django_db +def test_get_reference_deletes_when_llama_unavailable(monkeypatch): + from reference.exceptions import ReferenceLlamaUnavailableError + + def raise_unavailable(_block): + raise ReferenceLlamaUnavailableError( + "Reference Llama service unavailable: 404 Client Error" + ) + yield # pragma: no cover + + monkeypatch.setattr("reference.data_utils.mark_references", raise_unavailable) + + reference = Reference.objects.create( + mixed_citation="Smith J. Nature. 2024.", + status=ReferenceStatus.CREATING, + ) + ref_id = reference.id + + with pytest.raises(ReferenceLlamaUnavailableError, match="404"): + get_reference(ref_id) + + assert not Reference.objects.filter(id=ref_id).exists() + assert ElementCitation.objects.filter(reference_id=ref_id).count() == 0 + + +@pytest.mark.django_db +def test_resolve_references_result_omits_non_references(monkeypatch): + def fake_mark_reference_texts(texts): + out = [] + for text in texts: + if text.startswith("Figure"): + out.append(json.dumps({"is_reference": False})) + else: + out.append(json.dumps({"reftype": "journal", "title": text})) + return out + + monkeypatch.setattr( + "reference.data_utils.mark_reference_texts", + fake_mark_reference_texts, + ) + + results = resolve_references_result( + [ + "Smith J. Nature. 2024.", + "Figure 1. Map of the study area.", + "Doe A. Science. 2023.", + ] + ) + + assert [item["mixed_citation"] for item in results] == [ + "Smith J. Nature. 2024.", + "Doe A. Science. 2023.", + ] + assert ( + Reference.objects.filter( + mixed_citation="Figure 1. Map of the study area." + ).count() + == 0 + ) + assert Reference.objects.count() == 2 + + +@pytest.mark.django_db +def test_get_reference_deletes_when_only_non_reference(monkeypatch): + monkeypatch.setattr( + "reference.data_utils.mark_references", + lambda _block: iter( + [ + { + "references": "Figure 1. Caption.", + "choices": [json.dumps({"is_reference": False})], + } + ] + ), + ) + reference = Reference.objects.create( + mixed_citation="Figure 1. Caption.", + status=ReferenceStatus.CREATING, + ) + ref_id = reference.id + + get_reference(ref_id) + + assert not Reference.objects.filter(id=ref_id).exists() + assert ElementCitation.objects.count() == 0 + + +@pytest.mark.django_db +def test_get_reference_skips_non_reference_among_valid(monkeypatch): + def fake_mark_references(reference_block): + for ref_row in parse_reference_list(reference_block): + if ref_row.startswith("Figure"): + yield { + "references": ref_row, + "choices": [json.dumps({"is_reference": False})], + } + else: + yield { + "references": ref_row, + "choices": [json.dumps({"reftype": "journal", "title": ref_row})], + } + + monkeypatch.setattr( + "reference.data_utils.mark_references", + fake_mark_references, + ) + reference = Reference.objects.create( + mixed_citation="Ref A\nFigure 1. Caption.\nRef B", + status=ReferenceStatus.CREATING, + ) + + get_reference(reference.id) + + reference.refresh_from_db() + assert reference.status == ReferenceStatus.READY + titles = list(reference.element_citation.values_list("marked", flat=True)) + assert [item["title"] for item in titles] == ["Ref A", "Ref B"] + + +@pytest.mark.django_db +def test_resolve_reference_result_returns_xml(monkeypatch): + monkeypatch.setattr( + "reference.data_utils.get_reference", + lambda obj_id: None, + ) + + reference = Reference.objects.create( + mixed_citation="XML Ref", + status=ReferenceStatus.READY, + ) + ElementCitation.objects.create( + reference=reference, + marked={"reftype": "journal"}, + marked_xml="", + ) + + result = resolve_reference_result("XML Ref", output_type="xml") + + assert result["data"] == "" + + +@pytest.mark.django_db +def test_resolve_references_result_processes_each_item(monkeypatch): + monkeypatch.setattr( + "reference.data_utils.get_reference", + lambda obj_id: None, + ) + + for citation, title in (("Ref A", "A"), ("Ref B", "B")): + reference = Reference.objects.create( + mixed_citation=citation, + status=ReferenceStatus.READY, + ) + ElementCitation.objects.create( + reference=reference, + marked={"reftype": "journal", "title": title}, + ) + + results = resolve_references_result(["Ref A", "Ref B"]) + + assert len(results) == 2 + assert results[0]["data"]["title"] == "A" + assert results[1]["data"]["title"] == "B" + + +@pytest.mark.django_db +def test_resolve_references_result_batches_uncached(monkeypatch): + calls = [] + + def fake_mark_reference_texts(texts): + calls.append(list(texts)) + return [json.dumps({"reftype": "journal", "title": text}) for text in texts] + + monkeypatch.setattr( + "reference.data_utils.mark_reference_texts", + fake_mark_reference_texts, + ) + + results = resolve_references_result(["Ref A", "Ref B", "Ref C"]) + + assert calls == [["Ref A", "Ref B", "Ref C"]] + assert [item["data"]["title"] for item in results] == ["Ref A", "Ref B", "Ref C"] + assert Reference.objects.count() == 3 + + +@pytest.mark.django_db +def test_resolve_references_result_empty_list(): + assert resolve_references_result([]) == [] + + +@pytest.mark.django_db +def test_get_reference_marks_multiline_mixed_citation(monkeypatch): + marked_calls = [] + + def fake_mark_references(reference_block): + marked_calls.append(reference_block) + for ref_row in parse_reference_list(reference_block): + yield { + "references": ref_row, + "choices": [json.dumps({"reftype": "journal", "title": ref_row})], + } + + monkeypatch.setattr("reference.data_utils.mark_references", fake_mark_references) + + reference = Reference.objects.create( + mixed_citation="Ref A\nRef B", + status=ReferenceStatus.CREATING, + ) + + get_reference(reference.id) + + reference.refresh_from_db() + assert marked_calls == ["Ref A\nRef B"] + assert reference.status == ReferenceStatus.READY + assert reference.element_citation.count() == 2 + assert reference.element_citation.first().marked["title"] == "Ref A" + + +@pytest.mark.django_db +def test_api_accepts_reference_list(monkeypatch): + monkeypatch.setattr( + "reference.api.v1.views.resolve_references_result", + lambda references, user=None, output_type="json": [ + { + "mixed_citation": citation, + "data": {"reftype": "journal", "title": citation}, + } + for citation in parse_reference_list(references) + ], + ) + + User = get_user_model() + user = User.objects.create_user(username="apiuser", password="pass") + client = APIClient() + client.force_authenticate(user=user) + + response = client.post( + "/api/v1/reference/", + data=json.dumps( + { + "references": ["Ref A", "Ref B"], + "type": "json", + } + ), + content_type="application/json", + ) + + assert response.status_code == 200 + payload = response.json() + assert len(payload["references"]) == 2 + assert payload["references"][0]["mixed_citation"] == "Ref A" + assert payload["references"][1]["data"]["title"] == "Ref B" + + +@pytest.mark.django_db +def test_api_returns_503_when_llama_unavailable(monkeypatch): + from reference.exceptions import ReferenceLlamaUnavailableError + + def raise_unavailable(*_args, **_kwargs): + raise ReferenceLlamaUnavailableError( + "Reference Llama service unavailable: 404 Client Error" + ) + + monkeypatch.setattr( + "reference.api.v1.views.resolve_references_result", + raise_unavailable, + ) + + User = get_user_model() + user = User.objects.create_user(username="apiuser_llama_down", password="pass") + client = APIClient() + client.force_authenticate(user=user) + + before_refs = Reference.objects.count() + + response = client.post( + "/api/v1/reference/", + data=json.dumps({"references": "Smith J. Nature. 2024.", "type": "json"}), + content_type="application/json", + ) + + assert response.status_code == 503 + assert "Llama model is not available" in response.json()["error"] + assert Reference.objects.count() == before_refs + + +@pytest.mark.django_db +def test_api_accepts_form_urlencoded(monkeypatch): + monkeypatch.setattr( + "reference.api.v1.views.resolve_references_result", + lambda references, user=None, output_type="json": [ + { + "mixed_citation": citation, + "data": {"reftype": "journal", "title": citation}, + } + for citation in parse_reference_list(references) + ], + ) + + User = get_user_model() + user = User.objects.create_user(username="apiuser_form", password="pass") + client = APIClient() + client.force_authenticate(user=user) + + response = client.post( + "/api/v1/reference/", + data={"references": "Ref A\nRef B", "type": "json"}, + format="multipart", + ) + + assert response.status_code == 200 + payload = response.json() + assert len(payload["references"]) == 2 + assert payload["references"][0]["mixed_citation"] == "Ref A" + + +@pytest.mark.django_db +def test_api_keeps_single_string_response(monkeypatch): + monkeypatch.setattr( + "reference.api.v1.views.resolve_references_result", + lambda references, user=None, output_type="json": [ + { + "mixed_citation": "Ref A", + "data": {"reftype": "journal"}, + } + ], + ) + + User = get_user_model() + user = User.objects.create_user(username="apiuser2", password="pass") + client = APIClient() + client.force_authenticate(user=user) + + response = client.post( + "/api/v1/reference/", + data=json.dumps( + { + "references": "Ref A", + "type": "json", + } + ), + content_type="application/json", + ) + + assert response.status_code == 200 + assert response.json()["message"] == "reference: {'reftype': 'journal'}" + + +@pytest.mark.django_db +def test_api_multiline_string_returns_reference_list(monkeypatch): + monkeypatch.setattr( + "reference.api.v1.views.resolve_references_result", + lambda references, user=None, output_type="json": [ + {"mixed_citation": citation, "data": {"reftype": "journal"}} + for citation in parse_reference_list(references) + ], + ) + + User = get_user_model() + user = User.objects.create_user(username="apiuser3", password="pass") + client = APIClient() + client.force_authenticate(user=user) + + response = client.post( + "/api/v1/reference/", + data=json.dumps( + { + "references": "Ref A\nRef B", + "type": "json", + } + ), + content_type="application/json", + ) + + assert response.status_code == 200 + payload = response.json() + assert "references" in payload + assert len(payload["references"]) == 2 + + +@pytest.mark.django_db +def test_api_list_with_xml_type(monkeypatch): + monkeypatch.setattr( + "reference.api.v1.views.resolve_references_result", + lambda references, user=None, output_type="json": [ + { + "mixed_citation": citation, + "data": f"{citation}", + } + for citation in parse_reference_list(references) + ], + ) + + User = get_user_model() + user = User.objects.create_user(username="apiuser4", password="pass") + client = APIClient() + client.force_authenticate(user=user) + + response = client.post( + "/api/v1/reference/", + data=json.dumps( + { + "references": ["Ref A", "Ref B"], + "type": "xml", + } + ), + content_type="application/json", + ) + + assert response.status_code == 200 + payload = response.json() + assert payload["references"][0]["data"].startswith("") + + +@pytest.mark.django_db +def test_api_jats_returns_ref_list(monkeypatch): + monkeypatch.setattr( + "reference.api.v1.views.resolve_references_result", + lambda references, user=None, output_type="json": [ + { + "mixed_citation": citation, + "data": ( + f'' + f"{citation}" + f"" + ), + } + for citation in parse_reference_list(references) + ], + ) + + User = get_user_model() + user = User.objects.create_user(username="apiuser_jats", password="pass") + client = APIClient() + client.force_authenticate(user=user) + + response = client.post( + "/api/v1/reference/", + data=json.dumps( + { + "references": ["Ref A", "Ref B"], + "type": "jats", + } + ), + content_type="application/json", + ) + + assert response.status_code == 200 + payload = response.json() + assert "ref_list" in payload + assert "" in payload["ref_list"] + assert 'id="B1"' in payload["ref_list"] + assert 'id="B2"' in payload["ref_list"] + assert "Ref A" in payload["ref_list"] + assert "Ref B" in payload["ref_list"] + assert "element-citation" in payload["ref_list"] + + +@pytest.mark.django_db +def test_api_rejects_invalid_type(monkeypatch): + User = get_user_model() + user = User.objects.create_user(username="apiuser_badtype", password="pass") + client = APIClient() + client.force_authenticate(user=user) + + response = client.post( + "/api/v1/reference/", + data=json.dumps({"references": ["Ref A"], "type": "html"}), + content_type="application/json", + ) + + assert response.status_code == 400 + assert "type" in response.json() + + +@pytest.mark.django_db +def test_api_rejects_empty_references(): + User = get_user_model() + user = User.objects.create_user(username="apiuser5", password="pass") + client = APIClient() + client.force_authenticate(user=user) + + response = client.post( + "/api/v1/reference/", + data=json.dumps({"references": [], "type": "json"}), + content_type="application/json", + ) + + assert response.status_code == 400 + assert response.json()["error"] == "No references provided" + + +@pytest.mark.django_db +def test_api_rejects_invalid_json(): + User = get_user_model() + user = User.objects.create_user(username="apiuser6", password="pass") + client = APIClient() + client.force_authenticate(user=user) + + response = client.post( + "/api/v1/reference/", + data="{invalid", + content_type="application/json", + ) + + assert response.status_code == 400 + assert "detail" in response.json() + + +@pytest.mark.django_db +def test_api_requires_authentication(): + client = APIClient() + + response = client.post( + "/api/v1/reference/", + data=json.dumps({"references": ["Ref A"], "type": "json"}), + content_type="application/json", + ) + + assert response.status_code == 401 diff --git a/reference/utils/__init__.py b/reference/utils/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/reference/utils/__init__.py @@ -0,0 +1 @@ + diff --git a/reference/utils/references.py b/reference/utils/references.py new file mode 100644 index 0000000..96a113b --- /dev/null +++ b/reference/utils/references.py @@ -0,0 +1,90 @@ +import logging +import os +import re +import tempfile +import zipfile + +from lxml import etree + +from reference.exceptions import DocxReferencesError + +logger = logging.getLogger(__name__) + +REFERENCE_HEADING_RE = re.compile( + r"^(?:\d+[.\)]\s*)?(?:references?|referências?|referencias?|" + r"bibliography|bibliografia)\s*$", + re.IGNORECASE, +) + + +def stz_norm(value): + return re.sub(r"\s+", " ", str(value or "").strip().lower()) + + +def parse_reference_list(references): + if references is None: + return [] + if isinstance(references, str): + return [line.strip() for line in references.split("\n") if line.strip()] + if isinstance(references, (list, tuple)): + return [str(item).strip() for item in references if str(item).strip()] + text = str(references).strip() + return [text] if text else [] + + +def extract_references_section(text): + if not text: + return "" + lines = str(text).split("\n") + start = None + for index, line in enumerate(lines): + if REFERENCE_HEADING_RE.match(line.strip()): + start = index + 1 + break + if start is None: + return "" + return "\n".join(line.strip() for line in lines[start:] if line.strip()) + + +def extract_text_from_docx(docx_path, limit_chars=30000): + logger.info("Text extractor: using zipfile for %s", os.path.basename(docx_path)) + with zipfile.ZipFile(docx_path) as archive: + xml_bytes = archive.read("word/document.xml") + + root = etree.fromstring(xml_bytes) + nsmap = {"w": "http://schemas.openxmlformats.org/wordprocessingml/2006/main"} + + paragraphs = [] + for paragraph in root.xpath("//w:p | //w:tc//w:p", namespaces=nsmap): + text = "".join( + t.text or "" for t in paragraph.xpath(".//w:t", namespaces=nsmap) + ) + text = text.strip() + if text: + paragraphs.append(text) + + result = "\n".join(paragraphs) + if limit_chars is not None: + result = result[:limit_chars] + logger.info("Text extractor: zipfile produced %d chars", len(result)) + return result + + +def references_from_docx_upload(uploaded): + tmp_path = None + try: + with tempfile.NamedTemporaryFile(suffix=".docx", delete=False) as tmp: + for chunk in uploaded.chunks(): + tmp.write(chunk) + tmp_path = tmp.name + text = extract_text_from_docx(tmp_path, limit_chars=None) + except Exception as exc: + raise DocxReferencesError("Could not read DOCX file") from exc + finally: + if tmp_path and os.path.exists(tmp_path): + os.unlink(tmp_path) + + references = extract_references_section(text) + if not parse_reference_list(references): + raise DocxReferencesError("No references section found in DOCX") + return references diff --git a/reference/wagtail_hooks.py b/reference/wagtail_hooks.py new file mode 100644 index 0000000..bd460f8 --- /dev/null +++ b/reference/wagtail_hooks.py @@ -0,0 +1,72 @@ +from django.contrib import messages +from django.http import HttpResponseRedirect +from django.utils.translation import gettext_lazy as _ +from wagtail.admin.panels import FieldPanel, InlinePanel, ObjectList +from wagtail.snippets.models import register_snippet +from wagtail.snippets.views.snippets import CreateView, SnippetViewSet + +from config.menu import get_menu_order +from reference.create_forms import ReferenceCreateAdminForm +from reference.data_utils import resolve_references_result +from reference.exceptions import ( + ReferenceLlamaDisabledError, + ReferenceLlamaMisconfiguredError, + ReferenceLlamaUnavailableError, +) +from reference.models import Reference + + +class ReferenceCreateView(CreateView): + def get_panel(self): + return ObjectList( + [ + FieldPanel("mixed_citation"), + FieldPanel("docx_file"), + InlinePanel("element_citation", label=_("Cited Elements")), + ], + base_form_class=ReferenceCreateAdminForm, + ).bind_to_model(self.model) + + def get_form_class(self): + return self.panel.get_form_class() + + def form_valid(self, form): + try: + results = resolve_references_result( + form.cleaned_data["mixed_citation"], + user=self.request.user, + output_type="json", + ) + except ( + ReferenceLlamaDisabledError, + ReferenceLlamaMisconfiguredError, + ReferenceLlamaUnavailableError, + ) as exc: + messages.error( + self.request, + _("Llama model is not available: %(error)s") % {"error": str(exc)}, + ) + return self.render_to_response(self.get_context_data(form=form)) + + if results: + messages.success( + self.request, + _("Marked %(count)s reference(s).") % {"count": len(results)}, + ) + + return HttpResponseRedirect(self.get_success_url()) + + +class ReferenceModelViewSet(SnippetViewSet): + model = Reference + add_view_class = ReferenceCreateView + menu_name = "reference" + menu_label = _("References") + menu_icon = "openquote" + menu_order = get_menu_order("reference") + exclude_from_explorer = False + list_per_page = 20 + add_to_admin_menu = True + + +register_snippet(ReferenceModelViewSet) diff --git a/requirements/base.txt b/requirements/base.txt index 6977f3c..4d713cf 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,4 +1,4 @@ -setuptools>=68.2.2,<82 +setuptools>=68.2.2,<81 whitenoise==6.12.0 # https://github.com/evansd/whitenoise redis==7.4.0 # https://github.com/redis/redis-py celery==5.3.6 # pyup: < 6.0 # https://github.com/celery/celery @@ -8,15 +8,19 @@ hiredis==2.2.3 # https://github.com/redis/hiredis-py # ------------------------------------------------------------------------------ django==6.0.5 django-environ==0.13.0 +djangorestframework==3.17.1 +djangorestframework-simplejwt==5.5.1 django-celery-beat==2.9.0 # https://github.com/celery/django-celery-beat django_celery_results==2.6.0 django-compressor==4.6.0 # https://github.com/django-compressor/django-compressor +lxml>=5.3.0 # Wagtail # ------------------------------------------------------------------------------ wagtail==7.4.2 wagtail-modeladmin==2.3.0 wagtail-autocomplete==0.12.0 +wagtail-json-widget # Packtools (SPS package validation) # ------------------------------------------------------------------------------ diff --git a/requirements/local.txt b/requirements/local.txt index fed4d03..df6a0b3 100644 --- a/requirements/local.txt +++ b/requirements/local.txt @@ -10,5 +10,4 @@ django-debug-toolbar # https://github.com/jazzband/django-debug-toolbar pytest==9.0.3 pytest-django==4.11.1 pytest-cov==7.1.0 -coverage==7.10.6 -django-coverage-plugin==3.1.0 \ No newline at end of file +coverage==7.10.6 \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index e38b3c6..8623330 100644 --- a/setup.cfg +++ b/setup.cfg @@ -34,21 +34,25 @@ django_settings_module = config.settings.test ignore_errors = True [coverage:run] -include = - core/* - core_settings/* - users/* +source = + core + core_settings + users + xml_manager + config + reference omit = + */migrations/* *migrations* *tests* */tests/* */templates/* -plugins = - django_coverage_plugin [coverage:report] -fail_under = 100 include = core/* core_settings/* users/* + xml_manager/* + config/* + reference/*