[mod] infinite_scroll as preference

* oscar theme: code from searx/plugins/infinite_scroll.py
* simple theme: new implementation

Co-authored-by: Markus Heiser <markus.heiser@darmarIT.de>

searx/plugins/infinite_scroll.py

@@ -1,9 +0,0 @@
-from flask_babel import gettext
-
-name = gettext('Infinite scroll')
-description = gettext('Automatically load next page when scrolling to bottom of current page')
-default_on = False
-preference_section = 'ui'
-
-js_dependencies = ('plugins/js/infinite_scroll.js',)
-css_dependencies = ('plugins/css/infinite_scroll.css',)

searx/preferences.py

@@ -394,6 +394,17 @@ class Preferences:
                     'False': False
                 }
             ),
+            'infinite_scroll': MapSetting(
+                settings['ui']['infinite_scroll'],
+                locked=is_locked('infinite_scroll'),
+                map={
+                    '': settings['ui']['infinite_scroll'],
+                    '0': False,
+                    '1': True,
+                    'True': True,
+                    'False': False
+                }
+            ),
             # fmt: on
         }
 

searx/settings.yml

@@ -169,7 +169,6 @@ outgoing:
 #   - 'Ahmia blacklist'  # activation depends on outgoing.using_tor_proxy
 #   # these plugins are disabled if nothing is configured ..
 #   - 'Hostname replace'  # see hostname_replace configuration below
-#   - 'Infinite scroll'
 #   - 'Open Access DOI rewrite'
 #   - 'Vim-like hotkeys'
 

searx/settings_defaults.py

@@ -186,6 +186,7 @@ SCHEMA = {
         'results_on_new_tab': SettingsValue(bool, False),
         'advanced_search': SettingsValue(bool, False),
         'query_in_title': SettingsValue(bool, False),
+        'infinite_scroll': SettingsValue(bool, False),
     },
     'preferences': {
         'lock': SettingsValue(list, []),

searx/static/plugins/js/infinite_scroll.js

@@ -1,40 +0,0 @@
-function hasScrollbar() {
-    var root = document.compatMode=='BackCompat'? document.body : document.documentElement;
-    return root.scrollHeight>root.clientHeight;
-}
-
-function loadNextPage() {
-    var formData = $('#pagination form:last').serialize();
-    if (formData) {
-        $('#pagination').html('<div class="loading-spinner"></div>');
-        $.ajax({
-            type: "POST",
-            url: $('#search_form').prop('action'),
-            data: formData,
-            dataType: 'html',
-            success: function(data) {
-                var body = $(data);
-                $('#pagination').remove();
-                $('#main_results').append('<hr/>');
-                $('#main_results').append(body.find('.result'));
-                $('#main_results').append(body.find('#pagination'));
-                if(!hasScrollbar()) {
-                    loadNextPage();
-                }
-            }
-        });
-    }
-}
-
-$(document).ready(function() {
-    var win = $(window);
-    if(!hasScrollbar()) {
-        loadNextPage();
-    }
-    win.scroll(function() {
-        $("#pagination button").css("visibility", "hidden");
-        if ($(document).height() - win.height() - win.scrollTop() < 150) {
-            loadNextPage();
-        }
-    });
-});

searx/static/themes/__common__/js/image_layout.js

@@ -29,7 +29,8 @@
     this.verticalMargin = verticalMargin;
     this.horizontalMargin = horizontalMargin;
     this.maxHeight = maxHeight;
-    this.isAlignDone = true;
+    this.trottleCallToAlign = null;
+    this.alignAfterThrotteling = false;
   }
 
   /**
@@ -72,12 +73,12 @@
         // not loaded image : make it square as _getHeigth said it
         imgWidth = height;
       }
-      img.style.width = imgWidth + 'px';
+      img.setAttribute('width', Math.round(imgWidth));
-      img.style.height = height + 'px';
+      img.setAttribute('height', Math.round(height));
-      img.style.marginLeft = this.horizontalMargin + 'px';
+      img.style.marginLeft = Math.round(this.horizontalMargin) + 'px';
-      img.style.marginTop = this.horizontalMargin + 'px';
+      img.style.marginTop = Math.round(this.horizontalMargin) + 'px';
-      img.style.marginRight = this.verticalMargin - 7 + 'px'; // -4 is the negative margin of the inline element
+      img.style.marginRight = Math.round(this.verticalMargin - 7) + 'px'; // -4 is the negative margin of the inline element
-      img.style.marginBottom = this.verticalMargin - 7 + 'px';
+      img.style.marginBottom = Math.round(this.verticalMargin - 7) + 'px';
       resultNode = img.parentNode.parentNode;
       if (!resultNode.classList.contains('js')) {
         resultNode.classList.add('js');
@@ -112,6 +113,23 @@
     }
   };
 
+  ImageLayout.prototype.throttleAlign = function () {
+    var obj = this;
+    if (obj.trottleCallToAlign) {
+      obj.alignAfterThrotteling = true;
+    } else {
+      obj.alignAfterThrotteling = false;
+      obj.align();
+      obj.trottleCallToAlign = setTimeout(function () {
+        if (obj.alignAfterThrotteling) {
+          obj.align();
+        }
+        obj.alignAfterThrotteling = false;
+        obj.trottleCallToAlign = null;
+      }, 20);
+    }
+  }
+
   ImageLayout.prototype.align = function () {
     var i;
     var results_selectorNode = d.querySelectorAll(this.results_selector);
@@ -141,9 +159,9 @@
     }
   };
 
-  ImageLayout.prototype.watch = function () {
+  ImageLayout.prototype._monitorImages = function () {
     var i, img;
-    var obj = this;
+    var objthrottleAlign = this.throttleAlign.bind(this);
     var results_nodes = d.querySelectorAll(this.results_selector);
     var results_length = results_nodes.length;
 
@@ -152,34 +170,53 @@
       event.originalTarget.src = w.searxng.static_path + w.searxng.theme.img_load_error;
     }
 
-    function throttleAlign () {
-      if (obj.isAlignDone) {
-        obj.isAlignDone = false;
-        setTimeout(function () {
-          obj.align();
-          obj.isAlignDone = true;
-        }, 100);
-      }
-    }
-
-    // https://developer.mozilla.org/en-US/docs/Web/API/Window/pageshow_event
-    w.addEventListener('pageshow', throttleAlign);
-    // https://developer.mozilla.org/en-US/docs/Web/API/FileReader/load_event
-    w.addEventListener('load', throttleAlign);
-    // https://developer.mozilla.org/en-US/docs/Web/API/Window/resize_event
-    w.addEventListener('resize', throttleAlign);
-
     for (i = 0; i < results_length; i++) {
       img = results_nodes[i].querySelector(this.img_selector);
-      if (img !== null && img !== undefined) {
+      if (img !== null && img !== undefined && !img.classList.contains('aligned')) {
-        img.addEventListener('load', throttleAlign);
+        img.addEventListener('load', objthrottleAlign);
         // https://developer.mozilla.org/en-US/docs/Web/API/GlobalEventHandlers/onerror
-        img.addEventListener('error', throttleAlign);
+        img.addEventListener('error', objthrottleAlign);
+        img.addEventListener('timeout', objthrottleAlign);
         if (w.searxng.theme.img_load_error) {
           img.addEventListener('error', img_load_error, {once: true});
         }
+        img.classList.add('aligned');
       }
     }
+  }
+
+  ImageLayout.prototype.watch = function () {
+    var objthrottleAlign = this.throttleAlign.bind(this);
+
+    // https://developer.mozilla.org/en-US/docs/Web/API/Window/pageshow_event
+    w.addEventListener('pageshow', objthrottleAlign);
+    // https://developer.mozilla.org/en-US/docs/Web/API/FileReader/load_event
+    w.addEventListener('load', objthrottleAlign);
+    // https://developer.mozilla.org/en-US/docs/Web/API/Window/resize_event
+    w.addEventListener('resize', objthrottleAlign);
+
+    this._monitorImages();
+
+    var obj = this;
+
+    let observer = new MutationObserver(entries => {
+      let newElement = false;
+      for (let i = 0; i < entries.length; i++) {
+        if (entries[i].addedNodes.length > 0 && entries[i].addedNodes[0].classList.contains('result')) {
+          newElement = true;
+          break;
+        }
+      }
+      if (newElement) {
+        obj._monitorImages();
+      }
+    });
+    observer.observe(d.querySelector(this.container_selector), {
+      childList: true,
+      subtree: true,
+      attributes: false,
+      characterData: false,
+    })
   };
 
   w.searxng.ImageLayout = ImageLayout;

searx/static/themes/oscar/gruntfile.js

@@ -78,7 +78,7 @@ module.exports = function(grunt) {
       }
     },
     jshint: {
-      files: ['gruntfile.js', 'src/js/*.js', '../__common__/js/image_layout.js'],
+      files: ['gruntfile.js', 'src/js/*.js'],  // files in __common__ are linted by es lint in simple theme
       options: {
         reporterOutput: "",
         esversion: 6,

searx/static/themes/oscar/src/js/01_init.js

@@ -19,6 +19,7 @@ window.searxng = (function(d) {
 
     return {
         autocompleter: script.getAttribute('data-autocompleter') === 'true',
+        infinite_scroll: script.getAttribute('data-infinite-scroll') === 'true',
         method: script.getAttribute('data-method'),
         translations: JSON.parse(script.getAttribute('data-translations'))
     };

searx/static/themes/oscar/src/js/infinite_scroll.js

@@ -0,0 +1,50 @@
+/**
+ * @license
+ * (C) Copyright Contributors to the SearXNG project.
+ * (C) Copyright Contributors to the searx project (2014 - 2021).
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+$(document).ready(function() {
+    function hasScrollbar() {
+        var root = document.compatMode=='BackCompat'? document.body : document.documentElement;
+        return root.scrollHeight>root.clientHeight;
+    }
+
+    function loadNextPage() {
+        var formData = $('#pagination form:last').serialize();
+        if (formData) {
+            $('#pagination').html('<div class="loading-spinner"></div>');
+            $.ajax({
+                type: "POST",
+                url: $('#search_form').prop('action'),
+                data: formData,
+                dataType: 'html',
+                success: function(data) {
+                    var body = $(data);
+                    $('#pagination').remove();
+                    $('#main_results').append('<hr/>');
+                    $('#main_results').append(body.find('.result'));
+                    $('#main_results').append(body.find('#pagination'));
+                    if(!hasScrollbar()) {
+                        loadNextPage();
+                    }
+                }
+            });
+        }
+    }
+
+    if (searxng.infinite_scroll) {
+        var win = $(window);
+        $("html").addClass('infinite_scroll');
+        if(!hasScrollbar()) {
+            loadNextPage();
+        }
+        win.on('scroll', function() {
+            if ($(document).height() - win.height() - win.scrollTop() < 150) {
+                loadNextPage();
+            }
+        });
+    }
+
+});

searx/static/themes/oscar/src/less/infinite_scroll.less

@@ -2,6 +2,7 @@
     0%   { transform: rotate(0deg) }
     100% { transform: rotate(360deg) }
 }
+
 .loading-spinner {
     animation-duration: 0.75s;
     animation-iteration-count: infinite;
@@ -14,6 +15,7 @@
     border-radius: 50% !important;
     margin: 0 auto;
 }
-#pagination button {
+
+html.infinite_scroll #pagination button {
 	visibility: hidden;
 }

searx/static/themes/oscar/src/less/logicodev-dark/oscar.less

@@ -4,6 +4,7 @@
 @import "../../../../__common__/less/result_templates.less";
 @import "../../less/result_templates.less";
 @import "../../less/preferences.less";
+@import "../infinite_scroll.less";
 @import "../../generated/pygments-logicodev.less";
 
 @stacked-bar-chart: rgb(213, 216, 215, 1);

searx/static/themes/oscar/src/less/logicodev/oscar.less

@@ -4,6 +4,7 @@
 @import "../../../../__common__/less/result_templates.less";
 @import "../../less/result_templates.less";
 @import "../../less/preferences.less";
+@import "../infinite_scroll.less";
 @import "../../generated/pygments-logicodev.less";
 
 @import "navbar.less";

searx/static/themes/oscar/src/less/pointhi/oscar.less

@@ -4,6 +4,7 @@
 @import "../../../../__common__/less/result_templates.less";
 @import "../../less/result_templates.less";
 @import "../../less/preferences.less";
+@import "../infinite_scroll.less";
 @import "../../generated/pygments-pointhi.less";
 
 @import "footer.less";

searx/static/themes/simple/src/js/main/00_toolkit.js

@@ -59,43 +59,45 @@ window.searxng = (function (w, d) {
     }
   };
 
-  searxng.http = function (method, url) {
+  searxng.http = function (method, url, data = null) {
-    var req = new XMLHttpRequest(),
+    return new Promise(function (resolve, reject) {
-      resolve = function () {},
+      try {
-      reject = function () {},
+        var req = new XMLHttpRequest();
-      promise = {
+        req.open(method, url, true);
-        then: function (callback) { resolve = callback; return promise; },
+        req.timeout = 20000;
-        catch: function (callback) { reject = callback; return promise; }
-      };
 
-    try {
+        // On load
-      req.open(method, url, true);
+        req.onload = function () {
+          if (req.status == 200) {
+            resolve(req.response, req.responseType);
+          } else {
+            reject(Error(req.statusText));
+          }
+        };
 
-      // On load
+        // Handle network errors
-      req.onload = function () {
+        req.onerror = function () {
-        if (req.status == 200) {
+          reject(Error("Network Error"));
-          resolve(req.response, req.responseType);
+        };
-        } else {
+
-          reject(Error(req.statusText));
+        req.onabort = function () {
+          reject(Error("Transaction is aborted"));
+        };
+
+        req.ontimeout = function () {
+          reject(Error("Timeout"));
         }
-      };
 
-      // Handle network errors
+        // Make the request
-      req.onerror = function () {
+        if (data) {
-        reject(Error("Network Error"));
+          req.send(data)
-      };
+        } else {
-
+          req.send();
-      req.onabort = function () {
+        }
-        reject(Error("Transaction is aborted"));
+      } catch (ex) {
-      };
+        reject(ex);
-
+      }
-      // Make the request
+    });
-      req.send();
-    } catch (ex) {
-      reject(ex);
-    }
-
-    return promise;
   };
 
   searxng.loadStyle = function (src) {
@@ -148,5 +150,16 @@ window.searxng = (function (w, d) {
     this.parentNode.classList.add('invisible');
   });
 
+  function getEndpoint () {
+    for (var className of d.getElementsByTagName('body')[0].classList.values()) {
+      if (className.endsWith('_endpoint')) {
+        return className.split('_')[0];
+      }
+    }
+    return '';
+  }
+
+  searxng.endpoint = getEndpoint();
+
   return searxng;
 })(window, document);

searx/static/themes/simple/src/js/main/infinite_scroll.js

@@ -0,0 +1,88 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+/* global searxng */
+
+searxng.ready(function () {
+  'use strict';
+
+  searxng.infinite_scroll_supported = (
+    'IntersectionObserver' in window &&
+    'IntersectionObserverEntry' in window &&
+    'intersectionRatio' in window.IntersectionObserverEntry.prototype);
+
+  if (searxng.endpoint !== 'results') {
+    return;
+  }
+
+  if (!searxng.infinite_scroll_supported) {
+    console.log('IntersectionObserver not supported');
+    return;
+  }
+
+  let d = document;
+  var onlyImages = d.getElementById('results').classList.contains('only_template_images');
+
+  function newLoadSpinner () {
+    var loader = d.createElement('div');
+    loader.classList.add('loader');
+    return loader;
+  }
+
+  function replaceChildrenWith (element, children) {
+    element.textContent = '';
+    children.forEach(child => element.appendChild(child));
+  }
+
+  function loadNextPage (callback) {
+    var form = d.querySelector('#pagination form.next_page');
+    if (!form) {
+      return
+    }
+    replaceChildrenWith(d.querySelector('#pagination'), [ newLoadSpinner() ]);
+    var formData = new FormData(form);
+    searxng.http('POST', d.querySelector('#search').getAttribute('action'), formData).then(
+      function (response) {
+        var nextPageDoc = new DOMParser().parseFromString(response, 'text/html');
+        var articleList = nextPageDoc.querySelectorAll('#urls article');
+        var paginationElement = nextPageDoc.querySelector('#pagination');
+        d.querySelector('#pagination').remove();
+        if (articleList.length > 0 && !onlyImages) {
+          // do not add <hr> element when there are only images
+          d.querySelector('#urls').appendChild(d.createElement('hr'));
+        }
+        articleList.forEach(articleElement => {
+          d.querySelector('#urls').appendChild(articleElement);
+        });
+        if (paginationElement) {
+          d.querySelector('#results').appendChild(paginationElement);
+          callback();
+        }
+      }
+    ).catch(
+      function (err) {
+        console.log(err);
+        var e = d.createElement('div');
+        e.textContent = searxng.translations.error_loading_next_page;
+        e.classList.add('dialog-error');
+        e.setAttribute('role', 'alert');
+        replaceChildrenWith(d.querySelector('#pagination'), [ e ]);
+      }
+    )
+  }
+
+  if (searxng.infinite_scroll && searxng.infinite_scroll_supported) {
+    const intersectionObserveOptions = {
+      rootMargin: "20rem",
+    };
+    const observedSelector = 'article.result:last-child';
+    const observer = new IntersectionObserver(entries => {
+      const paginationEntry = entries[0];
+      if (paginationEntry.isIntersecting) {
+        observer.unobserve(paginationEntry.target);
+        loadNextPage(() => observer.observe(d.querySelector(observedSelector), intersectionObserveOptions));
+      }
+    });
+    observer.observe(d.querySelector(observedSelector), intersectionObserveOptions);
+  }
+
+});

searx/static/themes/simple/src/js/main/preferences.js

@@ -2,6 +2,10 @@
 (function (w, d, searxng) {
   'use strict';
 
+  if (searxng.endpoint !== 'preferences') {
+    return;
+  }
+
   searxng.ready(function () {
     let engine_descriptions = null;
     function load_engine_descriptions () {
@@ -19,10 +23,8 @@
       }
     }
 
-    if (d.querySelector('body[class="preferences_endpoint"]')) {
+    for (const el of d.querySelectorAll('[data-engine-name]')) {
-      for (const el of d.querySelectorAll('[data-engine-name]')) {
+      searxng.on(el, 'mouseenter', load_engine_descriptions);
-        searxng.on(el, 'mouseenter', load_engine_descriptions);
-      }
     }
   });
 })(window, document, window.searxng);

searx/static/themes/simple/src/js/main/results.js

@@ -2,6 +2,10 @@
 (function (w, d, searxng) {
   'use strict';
 
+  if (searxng.endpoint !== 'results') {
+    return;
+  }
+
   searxng.ready(function () {
     searxng.image_thumbnail_layout = new searxng.ImageLayout('#urls', '#urls .result-images', 'img.image_thumbnail', 14, 6, 200);
     searxng.image_thumbnail_layout.watch();

searx/static/themes/simple/src/less/style.less

@@ -771,15 +771,19 @@ article[data-vim-selected].category-social {
   margin: 1rem @results-tablet-offset 0 @results-tablet-offset;
   display: grid;
   grid-template-columns: 100%;
-  grid-template-rows: min-content min-content 1fr min-content min-content;
+  grid-template-rows: min-content min-content min-content 1fr min-content;
   gap: 0;
   grid-template-areas:
     "corrections"
-    "urls"
     "answers"
     "sidebar"
+    "urls"
     "pagination";
 
+  #sidebar {
+    display: none;
+  }
+
   #urls {
     width: inherit;
     margin: 0;

searx/templates/oscar/base.html

@@ -100,6 +100,7 @@
     <script src="{{ url_for('static', filename='js/searxng.min.js') }}"
             data-method="{{ method or 'POST' }}"
             data-autocompleter="{% if autocomplete %}true{% else %}false{% endif %}"
+            data-infinite-scroll="{% if infinite_scroll %}true{% else %}false{% endif %}"
             data-translations="{{ translations }}"></script>
     {% for script in scripts %}
     {{""}}<script src="{{ url_for('static', filename=script) }}"></script>

searx/templates/oscar/preferences.html

@@ -248,6 +248,17 @@
                         {{ preferences_item_footer(info, label, rtl) }}
                         {% endif %}
 
+                        {% if 'infinite_scroll' not in locked_preferences %}
+                        {% set label = _('Infinite scroll') %}
+                        {% set info = _('Automatically load next page when scrolling to bottom of current page') %}
+                        {{ preferences_item_header(info, label, rtl, 'infinite_scroll') }}
+                            <select class="form-control {{ custom_select_class(rtl) }}" name="infinite_scroll" id="infinite_scroll">
+                                <option value="1" {% if infinite_scroll %}selected="selected"{% endif %}>{{ _('On') }}</option>
+                                <option value="0" {% if not infinite_scroll %}selected="selected"{% endif %}>{{ _('Off')}}</option>
+                            </select>
+                        {{ preferences_item_footer(info, label, rtl) }}
+                        {% endif %}
+
                         {{ plugin_of_category('ui' )}}
                     </div>
                 </fieldset>

searx/templates/simple/base.html

@@ -23,7 +23,7 @@
           data-method="{{ method or 'POST' }}"
           data-autocompleter="{% if autocomplete %}true{% else %}false{% endif %}"
           data-search-on-category-select="{{ 'true' if 'plugins/js/search_on_category_select.js' in scripts else 'false'}}"
-          data-infinite-scroll="{{ 'true' if 'plugins/js/infinite_scroll.js' in scripts else 'false' }}"
+          data-infinite-scroll="{% if infinite_scroll %}true{% else %}false{% endif %}"
           data-hotkeys="{{ 'true' if 'plugins/js/vim_hotkeys.js' in scripts else 'false' }}"
           data-static-path="{{ url_for('static', filename='themes/simple') }}/"
           data-translations="{{ translations }}"></script>

searx/templates/simple/preferences.html

@@ -226,6 +226,18 @@
       <div class="description">{{_('Open result links on new browser tabs') }}</div>
     </fieldset>
     {% endif %}
+    {% if 'infinite_scroll' not in locked_preferences %}
+    <fieldset>
+      <legend>{{ _('Infinite scroll') }}</legend>
+      <p class="value">
+        <select name='infinite_scroll'>
+          <option value="1" {% if infinite_scroll %}selected="selected"{% endif %}>{{ _('On') }}</option>
+          <option value="0" {% if not infinite_scroll %}selected="selected"{% endif %}>{{ _('Off')}}</option>
+        </select>
+      </p>
+      <div class="description">{{ _('Automatically load next page when scrolling to bottom of current page') }}</div>
+    </fieldset>
+    {% endif %}
     {{ plugin_preferences('ui') }}
   {{ tab_footer() }}
 

searx/webapp.py

@@ -431,6 +431,8 @@ def get_translations():
         'no_item_found': gettext('No item found'),
         # /preferences: the source of the engine description (wikipedata, wikidata, website)
         'Source': gettext('Source'),
+        # infinite scroll
+        'error_loading_next_page': gettext('Error loading the next page'),
     }
 
 
@@ -463,6 +465,7 @@ def render(template_name: str, override_theme: str = None, **kwargs):
     kwargs['preferences'] = request.preferences
     kwargs['method'] = request.preferences.get_value('method')
     kwargs['autocomplete'] = request.preferences.get_value('autocomplete')
+    kwargs['infinite_scroll'] = request.preferences.get_value('infinite_scroll')
     kwargs['results_on_new_tab'] = request.preferences.get_value('results_on_new_tab')
     kwargs['advanced_search'] = request.preferences.get_value('advanced_search')
     kwargs['query_in_title'] = request.preferences.get_value('query_in_title')