diff --git a/LICENSE-EN.txt b/LICENSE-EN.txt index 07193d5..4234bf8 100644 --- a/LICENSE-EN.txt +++ b/LICENSE-EN.txt @@ -15,7 +15,7 @@ held by the respective copyright holders as noted in those files. Users are aske MIT License -Copyright (c) 2022 Government of Canada +Copyright (c) 2026 Government of Canada Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, diff --git a/LICENSE-FR.txt b/LICENSE-FR.txt index 70bee17..e569773 100644 --- a/LICENSE-FR.txt +++ b/LICENSE-FR.txt @@ -20,7 +20,7 @@ dans ces fichiers. Nous demandons aux utilisateurs de lire les licences des tier Licence MIT -(c) Droit d'auteur – Gouvernement du Canada, 2022 +(c) Droit d'auteur – Gouvernement du Canada, 2026 La présente autorise toute personne d'obtenir gratuitement une copie du présent logiciel et des documents connexes (le « logiciel »), de traiter le logiciel sans restriction, y compris, mais sans diff --git a/changes/62.changes b/changes/62.changes new file mode 100644 index 0000000..7f68989 --- /dev/null +++ b/changes/62.changes @@ -0,0 +1 @@ +Added additional comments to the core search application code and in a few cases removed dead or commented code. No functoinal changes. \ No newline at end of file diff --git a/oc_search/settings-sample.py b/oc_search/settings-sample.py index eda824f..749e6db 100644 --- a/oc_search/settings-sample.py +++ b/oc_search/settings-sample.py @@ -42,8 +42,9 @@ 'search', ] -## Optional applications +## Optional applications - this will be removed from search soon # 'ramp', + MIDDLEWARE = [ "corsheaders.middleware.CorsMiddleware", 'django.middleware.security.SecurityMiddleware', @@ -59,6 +60,8 @@ 'oc_search.middleware.CanadaBilingualMiddleware' ] +# General web app settings + CORS_ALLOW_ALL_ORIGINS = True CORS_REPLACE_HTTPS_REFERER = True SECURE_REFERRER_POLICY = 'unsafe-url' @@ -67,6 +70,10 @@ STATIC_ROOT = os.path.join(BASE_DIR, 'static') +WSGI_APPLICATION = 'oc_search.wsgi.application' + +JSON_DOWNLOADS_ALLOWED = False + # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/3.0/howto/static-files/ @@ -76,7 +83,6 @@ ('search_snippets', os.path.join(BASE_DIR, 'search', 'templates', 'snippets')), ('cache', os.path.join(BASE_DIR, 'cache')), ] -# ('ramp', os.path.join(BASE_DIR, 'ramp', 'viewer')), TEMPLATES = [ { @@ -95,8 +101,8 @@ }, ] -WSGI_APPLICATION = 'oc_search.wsgi.application' - +# Optional templates for Geoviewer - should be deprecated soon +# ('ramp', os.path.join(BASE_DIR, 'ramp', 'viewer')), # Database # https://docs.djangoproject.com/en/3.0/ref/settings/#databases @@ -125,8 +131,6 @@ # } # } -JSON_DOWNLOADS_ALLOWED = False - # Password validation # https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators @@ -188,6 +192,7 @@ 'level': 'INFO', }, } + # Django Cache settings CACHES = { @@ -270,17 +275,20 @@ 'table', 'thead', 'th', 'tr', 'tbody', 'td' ] MARKDOWN_FILTER_EXTRAS = ["tables", "break-on-newline"] + # These are IN ADDITION to the attributes defined in leach.sanitizer.ALLOWED_ATTRIBUTES + MARKDOWN_FILTER_ALLOWED_ATTRIBUTES = {'span': ['title', 'class'], "a": ["href", "title", "rel"]} +# Analytics options used by the open canada site + ADOBE_ANALYTICS_URL = '' GOOGLE_ANALYTICS_GTM_ID = '' GOOGLE_ANALYTICS_PROPERTY_ID = '' GOOGLE_ANALYTICS_GA4_ID = '' -IMPORT_EXPORT_USE_TRANSACTIONS = False +# Geoviewer Settings specifically for the Open Data Custom search - still required after RAMP app is removed -# Geoviewer Settings specifically for the Open Data Custom search OPEN_DATA_SOLR_SERVER_URL = "http://localhost:8983/solr" OPEN_DATA_CORE = "search_opendata" OPEN_DATA_BASE_URL_EN = "https://open.canada.ca/data/en/dataset/" @@ -290,7 +298,6 @@ OPEN_DATA_HOST_EN = "https://open.canada.ca" OPEN_DATA_HOST_FR = "https://ouvert.canada.ca" - # Celery Congifuration CELERY_BROKER_URL = 'redis://:@localhost:6379/0' diff --git a/oc_search/urls.py b/oc_search/urls.py index 0ac8167..285fd8a 100644 --- a/oc_search/urls.py +++ b/oc_search/urls.py @@ -27,11 +27,7 @@ path('search/admin/', admin.site.urls), ] -# urlpatterns += [ -# path('searchform//', include('search.urls')), -# path('rechercherformulaire//', include('search.urls')), -# ] - +# URLs for when Search is configured to use the URL path to determine language of web page if settings.SEARCH_LANG_USE_PATH: urlpatterns += [ path('', DefaultView.as_view(), name="HomePage"), @@ -68,6 +64,8 @@ path('search//', SearchFormView.as_view(), name="search_form"), path('rechercher//', SearchFormView.as_view(), name="search_form"), ] + +# URLs to use when Search is configured to use the hostname to determine the language of the webpage IE open.canada.ca vs ouvert.canada.ca else: urlpatterns += [ path('', DefaultView.as_view(), name="HomePage"), diff --git a/search/admin.py b/search/admin.py index 553994e..176a64e 100644 --- a/search/admin.py +++ b/search/admin.py @@ -5,7 +5,6 @@ # -- Searches --- - class SearchAdmin(admin.ModelAdmin): list_display = ['search_id', 'label_en', 'solr_core_name'] @@ -28,8 +27,7 @@ class SearchAdmin(admin.ModelAdmin): ('More-like-this', {'fields': ('mlt_enabled', 'mlt_items')}) ) - # -- Fields --- - +# -- Fields --- def make_facet_field(modeladmn, request, queryset): queryset.update(is_search_facet=True) diff --git a/search/extras/About_Extras.md b/search/extras/About_Extras.md new file mode 100644 index 0000000..3e55bd9 --- /dev/null +++ b/search/extras/About_Extras.md @@ -0,0 +1,5 @@ +This directory can be used to hold any extra files or data used by custom searches that does not +fit into the usual custom search format. It allows non-standard files to be distributed with +a custom search. + +Most proactive disclosure searches did not use this feature \ No newline at end of file diff --git a/search/models/custom/readme.md b/search/models/custom/readme.md index e3fa3de..6e524b4 100644 --- a/search/models/custom/readme.md +++ b/search/models/custom/readme.md @@ -1,3 +1,4 @@ -This directory holds custom Django database models used by custom searches. These models are dynamically loaded +This directory holds custom Django database models to be added to Django search application by custom searches. +These models are dynamically loaded into the search application but are not automatically migrated. -into the search application but are not automatically migrated. +Use the standard Django database migration commands to add these models to the database. diff --git a/search/query.py b/search/query.py index 1bb17cb..1cee8b6 100644 --- a/search/query.py +++ b/search/query.py @@ -48,7 +48,8 @@ def uuid_pattern(version): def calc_pagination_range(num_found: int, pagesize, current_page, delta=2): - # @TODO This is not very efficient - could be refactored + """ Calculate pagination values based on provided values """ + pages = int(ceil(num_found / pagesize)) if current_page > pages: current_page = pages @@ -96,7 +97,7 @@ def calc_starting_row(page_num: str, rows_per_page=10): def get_search_terms(search_text: str): - # Get any search terms + """ Tokenize search terms""" tr = RegexpTokenizer(r'[^"\s]\S*|".+?"', gaps=False) @@ -110,6 +111,8 @@ def get_search_terms(search_text: str): def get_query_fields(query_lang: str, fields: dict): + """ Return a language specific list of query fields for a search """ + qf = ['id'] for f in fields: if fields[f].solr_field_lang in [query_lang, 'bi']: @@ -130,6 +133,7 @@ def get_query_fields(query_lang: str, fields: dict): def get_mlt_fields(request: HttpRequest, fields: dict): + """ Return a language specific list of query fields for a more-like-this search """ qf = ['id'] for f in fields: if fields[f].solr_field_lang in [request.LANGUAGE_CODE, 'bi']: @@ -210,16 +214,19 @@ def create_solr_query(request: HttpRequest, search: Search, fields: dict, Codes: known_fields[request_field] = keys[request_field][0].split('|') # If sort not specified in the request, then use the default + if 'sort' not in solr_query: solr_query['sort'] = default_sort # Sometimes, the sort order will be forced to the default value + if override_sort: solr_query['sort'] = default_sort solr_query['q.op'] = search.solr_default_op # Create a Solr query field list based on the Fields Model. Expand the field list where needed + solr_query['qf'] = get_query_fields(request.LANGUAGE_CODE, fields) if export: @@ -238,6 +245,7 @@ def create_solr_query(request: HttpRequest, search: Search, fields: dict, Codes: solr_query['fl'] = ",".join(ef) else: solr_query['fl'] = ",".join(solr_query['qf']) + if not export: solr_query['start'] = start_row solr_query['rows'] = rows @@ -263,7 +271,8 @@ def create_solr_query(request: HttpRequest, search: Search, fields: dict, Codes: 'hl.highlightMultiTerm': True, }) - # Set a default sort order + # Set a default sort order if one not included in the query + if 'sort' not in solr_query: solr_query['sort'] = 'score desc' @@ -288,14 +297,19 @@ def create_solr_query(request: HttpRequest, search: Search, fields: dict, Codes: ff.append(facet) solr_query['fq'] = fq solr_query['facet.field'] = ff + if export and solr_query['sort'] == "score desc": solr_query['sort'] = "id asc" + if search.solr_debugging: solr_query['debugQuery'] = True + return solr_query def create_solr_mlt_query(request: HttpRequest, search: Search, fields: dict, start_row: int, record_id: str): + """ Create a Solr More-like-this query """ + solr_query = { 'q': 'id:"{0}"'.format(record_id), 'mlt': True, @@ -312,10 +326,12 @@ def create_solr_mlt_query(request: HttpRequest, search: Search, fields: dict, st } return solr_query + def create_post_solr_query(request: HttpRequest, search: Search, fields: dict, Codes: dict, facets: list, start_row: int = 0, rows: int = 10, default_sort='score desc', override_sort=False, is_export: bool = False, reset_filters: bool = False): - + """ Create a Solr query based an an HTTP POST form request """ + # Look for known fields in the POST request known_fields = {} @@ -390,6 +406,7 @@ def create_post_solr_query(request: HttpRequest, search: Search, fields: dict, C solr_query['q.op'] = search.solr_default_op # Create a Solr query field list based on the Fields Model. Expand the field list where needed + solr_query['qf'] = get_query_fields(request.LANGUAGE_CODE, fields) if not is_export: diff --git a/search/templatetags/search_extras.py b/search/templatetags/search_extras.py index a1614e5..0da5c0e 100644 --- a/search/templatetags/search_extras.py +++ b/search/templatetags/search_extras.py @@ -252,7 +252,6 @@ def add_str(value: str, arg1: str): return value + arg1 - @register.filter('replace_pageno') def trim_left(value: str, arg: int): return value.replace("__page__", str(arg)) @@ -276,6 +275,7 @@ def ds_status_label(value: str): def int_format(value: int, formatstr: str): return formatstr.format(value) + @register.filter('normailize_id_value') def normailize_id_value(value: str): return value.replace(" ", "_").replace('é', 'e').replace('à', 'a') diff --git a/search/views.py b/search/views.py index 15e7277..31b1e6d 100644 --- a/search/views.py +++ b/search/views.py @@ -32,10 +32,13 @@ def iter_namespace(ns_pkg): # returned name an absolute name instead of a relative one. This allows # import_module to work without having to do additional modification to # the name. + # Used by the Search dynamic plug-in system return pkgutil.iter_modules(ns_pkg.__path__, ns_pkg.__name__ + ".") def log_search_results(request: HttpRequest, search_logger: logging.Logger, search_type: str = "", format: str = "html", page_type: str = "search", search_text: str = '', doc_count: int = 0, hostname = "None"): + """ When enabled, Search can write a log entry for every query. Should only be for SHORT-TERM use due to high volume. """ + qurl = parse.urlparse(request.get_full_path()) query_values = parse.parse_qs(request.META["QUERY_STRING"]) @@ -60,6 +63,8 @@ def log_search_results(request: HttpRequest, search_logger: logging.Logger, sear def get_error_context(search_type: str, lang: str, error_msg=""): + """ Return a basic Django template context that can be used by any error message page """ + return { "language": lang, "LANGUAGE_CODE": lang, @@ -87,6 +92,12 @@ def get_error_context(search_type: str, lang: str, error_msg=""): def get_search_path(search_defn: Search, lang): + """ Return the proper path for a given search accounting for the way Search is configured """ + + # Search can handle English/French language using two different modes. Which mode is used is specified in the settings file + # 1. By hostname. The language of the site is determined by the host name: Eg. open.canada.ca (EN), ouvert.canada.ca (FR) + # 2. By path. The language is determined by a path prefixt IE. /search/en (EN), /rechercher/fr (FR) + return_url = "" if settings.SEARCH_LANG_USE_PATH: @@ -101,29 +112,10 @@ def get_search_path(search_defn: Search, lang): return_url = parse.urljoin(settings.SEARCH_HOST_PATH, f'/{search_defn.search_alias_en}') return return_url -def cache_search_file(cached_filename: str, sr: SolrResponse, rows=100000): - - if len(sr.docs) == 0: - return False - if not os.path.exists(cached_filename): - with open(cached_filename, 'w', newline='', encoding='utf8') as csv_file: - cache_writer = csv.writer(csv_file, dialect='excel', quoting=csv.QUOTE_ALL) - headers = list(sr.docs[0]) - headers[0] = u'\N{BOM}' + headers[0] - cache_writer.writerow(headers) - c = 0 - for i in sr.docs: - if c > rows: - break - try: - cache_writer.writerow(i.values()) - c += 1 - except UnicodeEncodeError: - pass - return True - - class SearchView(View): + """ Default Django View controller object for HTTP GET based searches. Also parent object for most other search views. """ + + # Search, Field, and Code definitions searches = {} search_alias_en = {} @@ -148,7 +140,9 @@ class SearchView(View): def __init__(self): super().__init__() - # Original function copied from https://packaging.python.org/guides/creating-and-discovering-plugins/ + # Original function derived from https://packaging.python.org/guides/creating-and-discovering-plugins/ + # Load all the custom search plug-in codes + self.discovered_plugins = { name: importlib.import_module(name) for finder, name, ispkg @@ -159,7 +153,9 @@ def __init__(self): cache = caches['local'] self.hostname = platform.node() - # Load Search and Field configuration + + # Load Search and Field configuration from database and save to a memory cache + # Values are always retrieved from the cache. if cache.get('searches') is None: search_queryset = Search.objects.all() @@ -241,7 +237,8 @@ def __init__(self): self.display_fields_names_en = cache.get('display_fields_names_en') self.display_fields_names_fr = cache.get('display_fields_names_fr') - # Load Code configuration + # Load Code configurations from cache if available, otherwise from database + if cache.get('codes_en') is None: self.codes_en = {} self.codes_fr = {} @@ -263,7 +260,10 @@ def __init__(self): self.codes_en = cache.get('codes_en') self.codes_fr = cache.get('codes_fr') + def get_default_display_fields(self, lang: str, search_type: str): + """ If not using a search template, then this returns a lits of fields marked for display by default """ + display_field_name = {} for f in self.fields[search_type]: if self.fields[search_type][f].solr_field_lang in [lang, 'bi']: @@ -275,7 +275,9 @@ def get_default_display_fields(self, lang: str, search_type: str): continue return display_field_name + def default_context(self, request: HttpRequest, search_type: str, lang: str): + """ Provide a basic context for rendering any page template """ context = { "language": lang, "cdts_version": settings.CDTS_VERSION, @@ -310,6 +312,8 @@ def default_context(self, request: HttpRequest, search_type: str, lang: str): "view_type": "SearchView" } + # these values vary depending on how I18N is done - with host names or by path + if settings.HTTP_FORM_PROTOCOL == "get": utl_fragments = request.META["PATH_INFO"].split('/') utl_fragments = utl_fragments if utl_fragments[-2] == search_type else utl_fragments[:-2] @@ -334,7 +338,9 @@ def default_context(self, request: HttpRequest, search_type: str, lang: str): return context + def get(self, request: HttpRequest, lang='en', search_type=''): + """ Perform a search - search terms are passed as URL parameters """ lang = request.LANGUAGE_CODE @@ -348,6 +354,7 @@ def get(self, request: HttpRequest, lang='en', search_type=''): request.session['prev_search'] = request.build_absolute_uri() # Replace search_type alias with actual search type + if lang == 'fr': if search_type in self.search_alias_fr: search_type = self.search_alias_fr[search_type] @@ -355,6 +362,8 @@ def get(self, request: HttpRequest, lang='en', search_type=''): if search_type in self.search_alias_en: search_type = self.search_alias_en[search_type] + # Set up context for rendering search results + if search_type in self.searches and not self.searches[search_type].is_disabled: context = self.default_context(request, search_type, lang) context["search_item_snippet"] = self.searches[search_type].search_item_snippet @@ -372,7 +381,8 @@ def get(self, request: HttpRequest, lang='en', search_type=''): context['main_content_body_top_snippet'] = self.searches[search_type].main_content_body_top_snippet context["query_type"] = "GET" - # Get search drop in message: + # Get search drop-in message: + context["general_msg"] = "" if lang == 'en': search_msg_en, is_new = Setting.objects.get_or_create(key="search.searchpage.topmessage.en") @@ -388,13 +398,16 @@ def get(self, request: HttpRequest, lang='en', search_type=''): core_name = self.searches[search_type].solr_core_name # Get the search result boundaries + start_row, page = calc_starting_row(request.GET.get('page', 1), rows_per_page=self.searches[search_type].results_page_size) # Link to Reset the search + context['reset_path'] = f"{request.scheme}://{request.get_host()}{request.path}" - # Compose the Solr query + # Compose the Solr query based on the user's search request + facets = self.facets_fr[search_type] if lang == 'fr' else self.facets_en[search_type] reversed_facets = [] for facet in facets: @@ -419,6 +432,7 @@ def get(self, request: HttpRequest, lang='en', search_type=''): facets, start_row, self.searches[search_type].results_page_size, record_id='', export=False, highlighting=True, default_sort=default_sort_order, override_sort=new_text_search) + # If the solr_query contains an error, then there was a problem with the request and # a 400 error page should be returned instead. @@ -430,7 +444,8 @@ def get(self, request: HttpRequest, lang='en', search_type=''): self.logger.info(f"Invalid search query: {solr_query['error_search_path']}") return render(request, '400.html', error_context, status=400) - # Call plugin pre-solr-query if defined + # Call plugin pre-solr-query extension + search_type_plugin = 'search.plugins.{0}'.format(search_type) if search_type_plugin in self.discovered_plugins: context, solr_query = self.discovered_plugins[search_type_plugin].pre_search_solr_query( @@ -447,7 +462,8 @@ def get(self, request: HttpRequest, lang='en', search_type=''): try: solr_response = solr.query(core_name, solr_query, highlight=True) - # Call plugin post-solr-query if it exists + # Call plugin post-solr-query function + if search_type_plugin in self.discovered_plugins: context, solr_response = self.discovered_plugins[search_type_plugin].post_search_solr_query( context, @@ -469,6 +485,7 @@ def get(self, request: HttpRequest, lang='en', search_type=''): if len(facets) > 0: # Facet search results context['facets'] = solr_response.get_facets() + # Get the selected facets from the search URL selected_facets = {} @@ -476,13 +493,16 @@ def get(self, request: HttpRequest, lang='en', search_type=''): if request_field in self.fields[search_type] and request_field in context['facets']: selected_facets[request_field] = request.GET.get(request_field, "").split('|') context['selected_facets'] = selected_facets + # Provide human friendly facet labels to the web page and any custom snippets facets_custom_snippets = {} for f in context['facets']: context['facets'][f]['__label__'] = self.fields[search_type][f].label_fr if lang == 'fr' else self.fields[search_type][f].label_en context['facets'][f]['__sortorder__'] = self.fields[search_type][f].solr_facet_sort + # If the facet is a code and sorting by label, then the facet needs to be resorted if self.fields[search_type][f].solr_facet_sort == 'label': + # Create an inverted index of the facet values facet_values = {} for facet_value in context['facets'][f].keys(): @@ -493,6 +513,7 @@ def get(self, request: HttpRequest, lang='en', search_type=''): facet_values[self.codes_en[search_type][f][facet_value]] = facet_value elif facet_value not in context['system_facet_fields'] and facet_value != '-' and facet_value not in self.codes_en[search_type][f]: self.logger.info(f"Unknown facet_value {f}:{facet_value}") + # Sort the facet values - use French locale for sorting if lang == "fr": sorted_facet_values = sorted(facet_values.keys(), key=unidecode) @@ -512,10 +533,14 @@ def get(self, request: HttpRequest, lang='en', search_type=''): context['facets'] = [] context['selected_facets'] = [] + # Retrieve basic results. Note that highlighting is enabled, so retrieve the results with + # the highlighting mark-up injected into the response + context['total_hits'] = solr_response.num_found context['docs'] = solr_response.get_highlighting() # Prepare a dictionary of language appropriate sort options + sort_options = {} sort_labels = self.searches[search_type].results_sort_order_display_fr.split(',') if lang == 'fr' else self.searches[search_type].results_sort_order_display_en.split(',') if lang == 'fr': @@ -541,6 +566,7 @@ def get(self, request: HttpRequest, lang='en', search_type=''): else: # Calculate the pagination values for the bottom of the search results page + context['show_pagination'] = True context['previous_page'] = (1 if page == 1 else page - 1) last_page = (context['pagination'][len(context['pagination']) - 1] if len(context['pagination']) > 0 else 1) @@ -551,7 +577,9 @@ def get(self, request: HttpRequest, lang='en', search_type=''): context['next_page'] = next_page context['currentpage'] = page - # Recreate the query string portion of the paging URL with the correct page and sort params + # Recreate the query string portion of the paging URL with the correct page and sort params. + # Used by the pagintation links on the search page + get_params = request.GET.dict() get_params['page'] = "__page__" get_params['sort'] = context['sort'] @@ -561,24 +589,28 @@ def get(self, request: HttpRequest, lang='en', search_type=''): pgntn_url_querystr = "&".join(q_list).replace(' ', '+') context['pgntn_path'] = f"{request.scheme}://{request.get_host()}{request.path}?{pgntn_url_querystr}" + # Call plugin pre_render_search() function (Note API version differenes in the function calls) + if search_type_plugin in self.discovered_plugins and self.discovered_plugins[search_type_plugin].plugin_api_version() == 1.1: context, template = self.discovered_plugins[search_type_plugin].pre_render_search(context, - self.searches[search_type].page_template, - request, - lang, - self.searches[search_type], - self.fields[search_type], - self.codes_fr[search_type] if lang == 'fr' else self.codes_en[search_type]) + self.searches[search_type].page_template, + request, + lang, + self.searches[search_type], + self.fields[search_type], + self.codes_fr[search_type] if lang == 'fr' else self.codes_en[search_type]) elif search_type_plugin in self.discovered_plugins and self.discovered_plugins[search_type_plugin].plugin_api_version() >= 1.2: context, template = self.discovered_plugins[search_type_plugin].pre_render_search(context, - self.searches[search_type].page_template, - request, - lang, - self.searches[search_type], - self.fields[search_type], - self.codes_fr[search_type] if lang == 'fr' else self.codes_en[search_type], - view_type='search') + self.searches[search_type].page_template, + request, + lang, + self.searches[search_type], + self.fields[search_type], + self.codes_fr[search_type] if lang == 'fr' else self.codes_en[search_type], + view_type='search') + # Users can optionally get the search results as a JSON object instead of the normal HTML page + search_format = request.GET.get("search_format", "html") if search_format == 'json' and self.searches[search_type].json_response: full_facet_dict = {} @@ -609,8 +641,14 @@ def get(self, request: HttpRequest, lang='en', search_type=''): 'facets': full_facet_dict, 'selected_facets': context['selected_facets'] if context['selected_facets'] else []} return JsonResponse(doc_dict) + + # Results can be returned as raw Solr results as well + elif search_format == 'solr' and self.searches[search_type].raw_solr_response: return JsonResponse(solr_response.data) + + # Render and return search results page + else: json_link = str(request.get_full_path()) if json_link.endswith("/"): @@ -625,6 +663,9 @@ def get(self, request: HttpRequest, lang='en', search_type=''): return render(request, self.searches[search_type].page_template, context) except (ConnectionError, SolrError) as ce: return render(request, 'error.html', get_error_context(search_type, lang, ce.args[0])) + + # If the search is disabled, display a message instead + elif search_type in self.searches and self.searches[search_type].is_disabled: context = self.default_context(request, search_type, lang) context['label_en'] = self.searches[search_type].label_en @@ -637,9 +678,13 @@ def get(self, request: HttpRequest, lang='en', search_type=''): class RecordView(SearchView): + """ Extend the search view to render individual records """ def get(self, request: HttpRequest, lang='en', search_type='', record_id=''): - lang = request.LANGUAGE_CODE # Replace search_type alias with actual search type + lang = request.LANGUAGE_CODE + + # Replace search_type alias with actual search type. The alias allows for readable URLs + if lang == 'fr': if search_type in self.search_alias_fr: search_type = self.search_alias_fr[search_type] @@ -665,12 +710,12 @@ def get(self, request: HttpRequest, lang='en', search_type='', record_id=''): # Compose the Solr query facets = {} - solr_query = create_solr_query(request, self.searches[search_type], self.fields[search_type], self.codes_fr[search_type] if lang == 'fr' else self.codes_en[search_type], facets, start_row, 25, record_id) - # Call plugin pre-solr-query if defined + # Call plugin pre-record-solr-query() function + search_type_plugin = 'search.plugins.{0}'.format(search_type) if search_type_plugin in self.discovered_plugins: context, solr_query = self.discovered_plugins[search_type_plugin].pre_record_solr_query( @@ -685,7 +730,7 @@ def get(self, request: HttpRequest, lang='en', search_type='', record_id=''): # Query Solr solr_response = solr.query(core_name, solr_query) - # Call plugin post-solr-query if it exists + # Call plugin post_record_solr_query() function if search_type_plugin in self.discovered_plugins: context, solr_response = self.discovered_plugins[search_type_plugin].post_record_solr_query( context, @@ -705,6 +750,8 @@ def get(self, request: HttpRequest, lang='en', search_type='', record_id=''): # Add code information context['codes'] = self.codes_fr[search_type] if lang == 'fr' else self.codes_en[search_type] + # Retrieve the appropriate language fields for the record + display_fields = [] display_field_name = {} for f in self.fields[search_type]: @@ -737,16 +784,19 @@ def get(self, request: HttpRequest, lang='en', search_type='', record_id=''): context['next_page'] = next_page context['currentpage'] = page + # Call plugin pre_render_record() function + if search_type_plugin in self.discovered_plugins and self.discovered_plugins[search_type_plugin].plugin_api_version() >= 1.1: context, template = self.discovered_plugins[search_type_plugin].pre_render_record(context, - self.searches[search_type].record_template, - request, - lang, - self.searches[search_type], - self.fields[search_type], - self.codes_fr[search_type] if lang == 'fr' else self.codes_en[search_type]) + self.searches[search_type].record_template, + request, + lang, + self.searches[search_type], + self.fields[search_type], + self.codes_fr[search_type] if lang == 'fr' else self.codes_en[search_type]) if settings.SEARCH_LOGGING_ON: log_search_results(request, self.search_logger, search_type=search_type, format='html', page_type='record', search_text=record_id, doc_count=solr_response.num_found, hostname=self.hostname) + return render(request, self.searches[search_type].record_template, context) else: @@ -754,6 +804,7 @@ def get(self, request: HttpRequest, lang='en', search_type='', record_id=''): class ExportView(SearchView): + """ Extend the SearchView for the Download Search Results function. This page only used for submittig the request and redirects to the results page """ def __init__(self): super().__init__() @@ -761,17 +812,16 @@ def __init__(self): if not os.path.exists(self.cache_dir): os.mkdir(self.cache_dir) - def cache_search_results_file(self, cached_filename: str, sr: SolrResponse, rows=100000): - - return cache_search_file(cached_filename, sr, rows) - def post(self, request, *args, **kwargs): - # create a form instance and populate it with data from the request: + """ Accept a form submission with the user's search query, start the Celery task, and redirect to the export status page """ + lang = request.LANGUAGE_CODE search_type = request.POST.get('export_search') if search_type in self.searches: + # Create a full version of the Solr query currently being used by the server + solr = SolrClient(settings.SOLR_SERVER_URL) core_name = self.searches[search_type].solr_core_name facets = self.facets_fr[search_type] if lang == 'fr' else self.facets_en[search_type] @@ -779,7 +829,8 @@ def post(self, request, *args, **kwargs): self.codes_fr[search_type] if lang == 'fr' else self.codes_en[search_type], facets, 1, 0, record_id='', export=True) - # Call plugin pre-solr-query if defined + # Call the plugin pre_export_solr_query() function + search_type_plugin = 'search.plugins.{0}'.format(search_type) if search_type_plugin in self.discovered_plugins: solr_query = self.discovered_plugins[search_type_plugin].pre_export_solr_query( @@ -789,15 +840,18 @@ def post(self, request, *args, **kwargs): self.codes_fr[search_type] if lang == 'fr' else self.codes_en[search_type], facets) - request_url = request.POST.get('export_search_path') - ## OPEN 4055 - Pass in the list if Field codes so that the readable title can be user in the exported - ## csv file. We can't pass in DB objects, so build a simple dict + # Pass in the list if Field codes so that the readable title can be user in the exported + # csv file. We can't pass in DB objects, so build a simple dict + field_names = {} for field in self.fields[search_type]: if lang == 'fr': field_names[field] = self.fields[search_type][field].label_fr else: field_names[field] = self.fields[search_type][field].label_en + + # Start the Celery background task that does the does the export work + task = export_search_results_csv.delay(solr_query, lang, core_name, field_names) if settings.SEARCH_LANG_USE_PATH: if lang == 'fr': @@ -813,30 +867,25 @@ def post(self, request, *args, **kwargs): else: return render(request, '404.html', get_error_context(search_type, lang)) + def get(self, request: HttpRequest, lang='en', search_type=''): + """ Not recommended for Production. Performs the same task as the post() function. + Accept a form submission with the user's search query, start the Celery task, and redirect to the export status page + """ lang = request.LANGUAGE_CODE if search_type in self.searches: - # Check to see if a recent cached results exists and return that instead if it exists - # hashed_query = hashlib.sha1(request.GET.urlencode().encode('utf8')).hexdigest() - # cached_filename = os.path.join(self.cache_dir, "{0}_{1}.csv".format(hashed_query, lang)) - # if os.path.exists(cached_filename): - # # If the cached file is over 5 minutes old, just delete and continue. In future, will want this to be a asynchronous opertaion - # if time() - os.path.getmtime(cached_filename) > 600: - # os.remove(cached_filename) - # else: - # if settings.EXPORT_FILE_CACHE_URL == "": - # return FileResponse(open(cached_filename, 'rb'), as_attachment=True) - # else: - # return HttpResponseRedirect(settings.EXPORT_FILE_CACHE_URL + "{0}_{1}.csv".format(hashed_query, lang)) - solr = SolrClient(settings.SOLR_SERVER_URL) + # Create a full version of the Solr query currently being used by the server + core_name = self.searches[search_type].solr_core_name facets = self.facets_fr[search_type] if lang == 'fr' else self.facets_en[search_type] solr_query = create_solr_query(request, self.searches[search_type], self.fields[search_type], self.codes_fr[search_type] if lang == 'fr' else self.codes_en[search_type], facets, 1, 0, record_id='', export=True) - # Call plugin pre-solr-query if defined + + # Call the plugin pre_export_solr_query() function + search_type_plugin = 'search.plugins.{0}'.format(search_type) if search_type_plugin in self.discovered_plugins: solr_query = self.discovered_plugins[search_type_plugin].pre_export_solr_query( @@ -845,7 +894,9 @@ def get(self, request: HttpRequest, lang='en', search_type=''): self.searches[search_type], self.fields[search_type], self.codes_fr[search_type] if lang == 'fr' else self.codes_en[search_type], facets) - + + # Start the Celery background task that does the does the export work + task = export_search_results_csv.delay(solr_query, lang, core_name) if settings.SEARCH_LANG_USE_PATH: @@ -864,11 +915,16 @@ def get(self, request: HttpRequest, lang='en', search_type=''): class ExportStatusView(View): + """ Extend the SearchView object to return a the Celery task progress status for a given export. + For use with AJAX calls. Does not render a page unless there is an erroy, otherwise return a JSON status object. """ + def __init__(self): super().__init__() def get(self, request: HttpRequest, lang='en', search_type='', task_id=''): + """ Return the current Celery task status, or redirect to an error page. """ + translation.activate(lang) response_dict = {"file_url": ""} if task_id == "": @@ -878,8 +934,8 @@ def get(self, request: HttpRequest, lang='en', search_type='', task_id=''): try: # Note: as of Nov, 2022 the django-celery-results module only saves a task record # to the database AFTER the task has started, therefore the other status values are not - # used at this time, but are present in anticipation of this functionality being added to the - # module in the future. + # used at this time, but are present in case this functionality is added in the future. + task = TaskResult.objects.get(task_id=task_id) response_dict['task_status'] = task.status if task.status == "SUCCESS": @@ -909,6 +965,7 @@ def get(self, request: HttpRequest, lang='en', search_type='', task_id=''): class DownloadSearchResultsView(View): + """ Extend the SearchView object for the Export Status Page where users can download search results """ searches = {} search_alias_en = {} @@ -936,7 +993,10 @@ def __init__(self): self.search_alias_en = cache.get('search_alias_en') self.search_alias_fr = cache.get('search_alias_fr') + def get(self, request: HttpRequest, lang='en', search_type='', task_id=''): + """ Render the download results page. AJAX on page will poll for results until the task is complete. + When the task is down, a download link is displayed """ if settings.SEARCH_LANG_USE_PATH: subpaths = request.path.split('/') @@ -998,6 +1058,8 @@ def get(self, request: HttpRequest, lang='en', search_type='', task_id=''): context['download_status_url'] = f"{settings.SEARCH_HOST_PATH}/search-results/en/{search_type}/{task_id}" context['back_to_url'] = get_search_path(self.searches[search_type], lang) return render(request, 'download.html', context) + + # Do not make available if the search is disabled elif search_type in self.searches and self.searches[search_type].is_disabled: context = {"language": lang,} context['label_en'] = self.searches[search_type].label_en @@ -1010,16 +1072,14 @@ def get(self, request: HttpRequest, lang='en', search_type='', task_id=''): class MoreLikeThisView(SearchView): + """ Extend the SearchView object to display the results from a Solr More-Like-This request """ def __init__(self): super().__init__() - # @TODO In the query class, use a shared method for fl and create new mothod fr MLT queries - # Note, for MLT, use the standard query but specify MLT params id:XXXX mlt=true&mlt.fl=title_en,name_en,purpose_en - # http://localhost:8983/solr/core_travelq/select?mlt.count=5&mlt.fl=title_en%2Cname_en%2Cpurpose_en%2Cid&mlt=true&q=id%3A%22aafc-aac%2CT-2017-Q1-00003%22 - # http://127.0.0.1:8000/search/en/travelq/similar/aafc-aac,T-2017-Q1-00003 - def get(self, request: HttpRequest, lang='en', search_type='', record_id=''): + """ Render the search's More-like-This page. """ + lang = request.LANGUAGE_CODE # Replace search_type alias with actual search type @@ -1054,11 +1114,15 @@ def get(self, request: HttpRequest, lang='en', search_type='', record_id=''): core_name = self.searches[search_type].solr_core_name # Get the search result boundaries + start_row, page = calc_starting_row(request.GET.get('page', 1), rows_per_page=self.searches[search_type].mlt_items) + # Compose the Solr query + solr_query = create_solr_mlt_query(request, self.searches[search_type], self.fields[search_type], start_row, record_id) - # Call plugin pre-solr-query if defined + # Call the plugin pre_mlt_solr_query() function + search_type_plugin = 'search.plugins.{0}'.format(search_type) if search_type_plugin in self.discovered_plugins: context, solr_query = self.discovered_plugins[search_type_plugin].pre_mlt_solr_query( @@ -1072,6 +1136,8 @@ def get(self, request: HttpRequest, lang='en', search_type='', record_id=''): # Query Solr solr_response = solr.query(core_name, solr_query) + # Call the plugin post_mlt_solr_query() function + if search_type_plugin in self.discovered_plugins: context, solr_query = self.discovered_plugins[search_type_plugin].post_mlt_solr_query( context, @@ -1084,8 +1150,10 @@ def get(self, request: HttpRequest, lang='en', search_type='', record_id=''): context['docs'] = solr_response.data['moreLikeThis'][record_id]['docs'] context['original_doc'] = solr_response.docs[0] - template = "more_like_this.html" + + # Call the plugin pre_render_search() function. Note there are changes to this in version 1.2 API + if search_type_plugin in self.discovered_plugins and self.discovered_plugins[search_type_plugin].plugin_api_version() == 1.1: context, template = self.discovered_plugins[search_type_plugin].pre_render_search(context, self.searches[search_type].more_like_this_template, @@ -1105,6 +1173,7 @@ def get(self, request: HttpRequest, lang='en', search_type='', record_id=''): view_type="mlt") context['back_to_url'] = get_search_path(self.searches[search_type], lang) + return render(request, template, context) else: @@ -1112,6 +1181,7 @@ def get(self, request: HttpRequest, lang='en', search_type='', record_id=''): class HomeView(SearchView): + """ Extends the SearchView object to display of list installed custom searches """ def __init__(self): super().__init__() @@ -1142,6 +1212,7 @@ def get(self, request: HttpRequest, lang='en'): class PageView(SearchView): + """ Displays the help text from a static template """ def __init__(self): super().__init__() @@ -1195,6 +1266,8 @@ def get(self, request: HttpRequest, lang='en', page_type=''): class DefaultView(View): + """ When no search ID is specified, the search application will redirect the user to a + search specified in the settings file. """ def get(self, request: HttpRequest): if settings.SEARCH_LANG_USE_PATH: @@ -1222,18 +1295,21 @@ def get(self, request: HttpRequest): class SearchFormView(SearchView): + """ Default Django View controller object for HTTP POST based searches. """ non_filter_fields = ['page', 'sort', 'export_search', 'export_search_path', 'csrfmiddlewaretoken', 'search', 'clearsearch'] non_facet_fields = ['search_text', 'page', 'sort', 'export_search', 'export_search_path', 'csrfmiddlewaretoken', 'search', 'clearsearch'] + def __init__(self): super().__init__() self.cache_dir = settings.EXPORT_FILE_CACHE_DIR if not os.path.exists(self.cache_dir): os.mkdir(self.cache_dir) + def de_alias(self, lang, search_type): - # Replace search_type alias with actual search type + """ Replace friendly readable search_type alias with actual search object type ID """ if lang == 'fr': if search_type in self.search_alias_fr: @@ -1243,9 +1319,11 @@ def de_alias(self, lang, search_type): return self.search_alias_en[search_type] return search_type + def to_solr_query(self, request: HttpRequest, search_type: str, lang: str, start_row: int, num_rows: int, is_export: bool, reset_filters: bool): - # Generate the Solr query for both the first GET and subsequent POST requests + """ Generate the Solr query for both the first GET and subsequent POST requests """ + # First view of a search page is always a GET request, not a form request, so if GET, assume a generic query if request.method == "GET": default_sort_order = self.searches[search_type].results_sort_default_fr if self.searches[ search_type].results_sort_default_fr else 'score desc' @@ -1282,8 +1360,9 @@ def to_solr_query(self, request: HttpRequest, search_type: str, lang: str, start reset_filters=reset_filters) return solr_query + def get_search_terms(self, search_text: str): - # Get any search terms + """ Tokenize search terms """ tr = RegexpTokenizer(r'[^"\s]\S*|".+?"', gaps=False) @@ -1295,8 +1374,9 @@ def get_search_terms(self, search_text: str): solr_search_terms = ' '.join(search_terms) return solr_search_terms + def default_search_context(self, context: dict, lang: str, search_type: str, request: HttpRequest): - # Basic page context information for all situations + """ Return basic page context information for all situations """ form_cxt = { "search_item_snippet": self.searches[search_type].search_item_snippet, @@ -1318,19 +1398,24 @@ def default_search_context(self, context: dict, lang: str, search_type: str, req } return {**context, **form_cxt} + def get(self, request, *args, **kwargs): + """ Django hook for GET style page request. """ request.session['query_type'] = "GET" return self.search_page(request, *args, **kwargs) + def post(self, request, *args, **kwargs): + """ Django hook for POST style page request. """ request.session['query_type'] = "POST" return self.search_page(request, *args, **kwargs) - def search_page(self, request: HttpRequest, *args, **kwargs): - # NOTE for Django developers - we are doing our own form handling and not using the Django forms + def search_page(self, request: HttpRequest, *args, **kwargs): + """ Handle search queries by querying Solr and rendering the results """ + # NOTE for Django developers - OCS does its own form handling and does not use Django model forms # Replace search_type alias with actual search type @@ -1518,9 +1603,6 @@ def search_page(self, request: HttpRequest, *args, **kwargs): context['total_hits'] = solr_response.num_found context['docs'] = solr_response.get_highlighting() - # @TODO need to set up a JSON link - # json_format_url - # Prepare a dictionary of language appropriate sort options sort_options = {}