[FEAT] Adding robots.txt rules (/api/ should not be known and we do not...

[FEAT] Adding robots.txt rules (/api/ should not be known and we do not divulgate this informations to malwares); check the logs to improve it.

gargantext/views/pages/main.py

@@ -39,6 +39,15 @@ def about(request):
         },
     )
 
+def robots(request):
+    '''About Gargantext, its team and sponsors
+    '''
+    return render(
+        template_name = 'pages/main/robots.txt',
+        request = request,
+        context = {},
+    )
+
 
 def maintenance(request):
     '''Gargantext out of service

gargantext/views/pages/urls.py

@@ -10,6 +10,7 @@ urlpatterns = [
     # presentation pages
     url(r'^$', main.home),
     url(r'^about/?$', main.about),
+    url(r'^robots.txt$', main.robots),
     # maintenance mode
     url(r'^maintenance/?$', main.maintenance),
     # authentication