add robots txt

core/urls.py

@@ -1,9 +1,11 @@
 from django.urls import path
+from django.shortcuts import HttpResponse
 from . import views
 
 
 urlpatterns = [
     path('', views.home, name='home'),
+    path('robots.txt', lambda x: HttpResponse("User-Agent: *\nDisallow: /", content_type="text/plain"), name="robots_file"),
     path('health/', views.health, name='health'),
     path('shipping/edit/', views.edit_shipping, name='edit_shipping'),
 ]