Skip to content

CIRCLE / cloud

Go to a project

A prog2-höz tartozó friss repo anyagok itt elérhetőek: https://git.iit.bme.hu/

Commit 9b3516eb by Your Name
Options

extended storage handling

parent b578176f
Showing with 353 additions and 75 deletions
circle/dashboard/forms.py
......@@ -728,48 +728,58 @@ class LeaseForm(forms.ModelForm):
Field("delete_interval_seconds", type="hidden", value="0"),
HTML(string_concat("<label>", _("Suspend in"), "</label>")),
Div(
NumberField("suspend_minutes", css_class="form-control"),
Div(
HTML(_("min")),
css_class="input-group-addon",
),
NumberField("suspend_hours", css_class="form-control"),
Div(
HTML(_("hours")),
HTML(_("h")),
css_class="input-group-addon",
),
NumberField("suspend_days", css_class="form-control"),
Div(
HTML(_("days")),
HTML(_("d")),
css_class="input-group-addon",
),
NumberField("suspend_weeks", css_class="form-control"),
Div(
HTML(_("weeks")),
HTML(_("wk")),
css_class="input-group-addon",
),
NumberField("suspend_months", css_class="form-control"),
Div(
HTML(_("months")),
HTML(_("mo")),
css_class="input-group-addon",
),
css_class="input-group interval-input",
),
HTML(string_concat("<label>", _("Delete in"), "</label>")),
Div(
NumberField("delete_minutes", css_class="form-control"),
Div(
HTML(_("min")),
css_class="input-group-addon",
),
NumberField("delete_hours", css_class="form-control"),
Div(
HTML(_("hours")),
HTML(_("h")),
css_class="input-group-addon",
),
NumberField("delete_days", css_class="form-control"),
Div(
HTML(_("days")),
HTML(_("d")),
css_class="input-group-addon",
),
NumberField("delete_weeks", css_class="form-control"),
Div(
HTML(_("weeks")),
HTML(_("wk")),
css_class="input-group-addon",
),
NumberField("delete_months", css_class="form-control"),
Div(
HTML(_("months")),
HTML(_("mo")),
css_class="input-group-addon",
),
css_class="input-group interval-input",
......@@ -1706,10 +1716,10 @@ class UserListSearchForm(forms.Form):
class DataStoreForm(ModelForm):
@property
def helper(self):
helper = FormHelper()
helper.form_tag = False # IMPORTANT: form tag is in template
helper.layout = Layout(
Fieldset(
'',
......@@ -1717,15 +1727,12 @@ class DataStoreForm(ModelForm):
'path',
'hostname',
),
FormActions(
Submit('submit', _('Save')),
)
)
return helper
class Meta:
model = DataStore
fields = ("name", "path", "hostname",)
fields = ("name", "path", "hostname")
class DiskForm(ModelForm):
......
circle/dashboard/static/dashboard/activity.js
......@@ -104,9 +104,19 @@ $(function() {
});
/* if the operation fails show the modal again */
/* submit only the operation form via AJAX (JSON response expected) */
$("body").on("submit", "#confirmation-modal form", function (e) {
e.preventDefault()
var $form = $(this);
// Only intercept the modal "operation" form submission.
// If the form does NOT contain the op submit button, let the browser handle it normally
// (redirects, GET forms, multi-step forms, etc.).
if ($form.find("#op-form-send").length === 0) {
return true;
}
e.preventDefault();
var url = $form.attr("action");
$.ajax({
......@@ -115,10 +125,9 @@ $(function() {
type: 'POST',
data: $form.serialize(),
success: function (data) {
// mindig zárjuk le az aktuális modalt
$('#confirmation-modal').modal("hide");
if (data.success) {
if (data && data.success) {
$('a[href="#activity"]').trigger("click");
if (data.with_reload) {
......@@ -129,7 +138,6 @@ $(function() {
addMessage(data.messages.join("<br />"), data.success ? "success" : "danger");
}
} else {
// a bezárás után nyissuk meg az új (hibás) modalt
$('#confirmation-modal').one('hidden.bs.modal', function () {
showConfirmationModal(data);
});
......@@ -140,6 +148,8 @@ $(function() {
if (xhr.status === 500) {
addMessage("500 Internal Server Error", "danger");
} else if (xhr.status === 405) {
addMessage("405 Method Not Allowed", "danger");
} else {
addMessage(xhr.status + " Unknown Error", "danger");
}
......
circle/dashboard/templates/base.html
......@@ -99,5 +99,100 @@
{% block extra_etc %}
{% endblock %}
<style>
/* Full-page loading overlay */
#loading-overlay {
position: fixed;
top: 0; left: 0;
width: 100%;
height: 100%;
background: rgba(255, 255, 255, 0.85);
z-index: 9999;
display: none;
}
#loading-overlay .spinner {
position: absolute;
top: 50%;
left: 50%;
transform: translate(-50%, -50%);
text-align: center;
color: #555;
}
</style>
<div id="loading-overlay">
<div class="spinner">
<i class="fa fa-spinner fa-spin fa-3x"></i>
<p>{% trans "Loading..." %}</p>
</div>
</div>
// <script>
// (function () {
// // Robust loader without beforeunload/select-change pitfalls.
// var overlay = document.getElementById("loading-overlay");
// if (!overlay) return;
//
// var timer = null;
//
// function showLoaderDelayed() {
// // Avoid stacking timers.
// if (timer) return;
//
// timer = setTimeout(function () {
// overlay.style.display = "block";
// }, 300); // show only if navigation takes noticeable time
// }
//
// function cancelLoader() {
// if (timer) {
// clearTimeout(timer);
// timer = null;
// }
// // In case it was shown and the browser restored from BFCache.
// overlay.style.display = "none";
// }
//
// // Cancel on load/pageshow (covers bfcache restore too).
// window.addEventListener("load", cancelLoader);
// window.addEventListener("pageshow", cancelLoader);
//
// // Show on form submit (POST or GET).
// var forms = document.getElementsByTagName("form");
// for (var i = 0; i < forms.length; i++) {
// forms[i].addEventListener("submit", function () {
//// showLoaderDelayed();
// });
// }
// // Show on autosubmit select change (programmatic form.submit() does not trigger submit events).
// var autos = document.querySelectorAll('select[data-autosubmit="1"]');
// for (var k = 0; k < autos.length; k++) {
// autos[k].addEventListener("change", function () {
// showLoaderDelayed();
// });
// }
//
// // Show on same-tab link clicks.
// var links = document.getElementsByTagName("a");
// for (var j = 0; j < links.length; j++) {
// (function (a) {
// a.addEventListener("click", function (e) {
// // Ignore modified clicks (new tab/window, etc.)
// if (e.ctrlKey || e.shiftKey || e.metaKey || e.altKey) return;
// if (a.target && a.target !== "_self") return;
// if (!a.href) return;
//
// // Ignore hash-only navigation on the same page.
// var href = a.getAttribute("href");
// if (href && href.charAt(0) === "#") return;
//
//// showLoaderDelayed();
// });
// })(links[j]);
// }
// })();
// </script>
</body>
</html>
circle/dashboard/templates/dashboard/lease-edit.html
......@@ -6,7 +6,7 @@
{% block content %}
<div class="row">
<div class="col-md-7">
<div class="col-md-6">
<div class="panel panel-default">
<div class="panel-heading">
<a class="pull-right btn btn-default btn-xs" href="{% url "dashboard.views.template-list" %}">{% trans "Back" %}</a>
......@@ -21,7 +21,7 @@
</div>
</div>
<div class="col-md-5">
<div class="col-md-6">
<div class="panel panel-default">
<div class="panel-heading">
<h4 class="no-margin"><i class="icon-group"></i> {% trans "Manage access" %}</h4>
......
circle/dashboard/templates/dashboard/storage/detail.html
......@@ -16,10 +16,51 @@
<h3 class="no-margin"><i class="fa fa-database"></i> {% trans "Datastore" %}</h3>
</div>
<div class="panel-body">
{% crispy form %}
<form method="get" action="">
<div class="form-group">
<label>{% trans "Datastore" %}</label>
<select class="form-control" name="ds" data-autosubmit="1" onchange="this.form.submit()">
<option value="new" {% if ds_selected == "new" %}selected{% endif %}>
-- {% trans "Create new datastore" %} --
</option>
{% for d in datastores %}
<option value="{{ d.pk }}" {% if ds and ds.pk == d.pk %}selected{% endif %}>
{{ d.name }} — {{ d.hostname }} : {{ d.path }}
</option>
{% endfor %}
</select>
</div>
{# tartsuk meg a filter/search paramokat, hogy ne vesszenek el #}
<input type="hidden" name="filter" value="{{ request.GET.filter }}">
<input type="hidden" name="s" value="{{ request.GET.s }}">
</form>
<hr/>
<form method="post" action="">
{% csrf_token %}
<input type="hidden" name="ds" value="{{ ds_selected }}">
{% crispy form %}
<div class="form-actions">
{% if mode == "create" %}
<button type="submit" class="btn btn-success">
<i class="fa fa-plus"></i> {% trans "Create" %}
</button>
{% else %}
<button type="submit" class="btn btn-primary">
<i class="fa fa-save"></i> {% trans "Save" %}
</button>
{% endif %}
</div>
</form>
</div><!-- .panel-body -->
</div>
</div>
{% if stats %}
<div class="col-md-7">
<div class="panel panel-default">
<div class="panel-heading">
......@@ -62,10 +103,12 @@
</div><!-- .panel-body -->
</div>
</div>
{% endif %}
</div>
<div class="row">
<div class="col-md-12">
{% if stats %}
<div class="panel panel-default">
<div class="panel-heading">
<h3 class="no-margin"><i class="fa fa-file"></i> {% trans "Disks" %}</h3>
......@@ -103,11 +146,13 @@
</div>
</div><!-- .panel-body -->
</div>
{% endif %}
</div>
</div>
<div class="row">
<div class="col-md-12">
{% if stats %}
<div class="panel panel-default">
<div class="panel-heading">
<h3 class="no-margin">
......@@ -152,6 +197,7 @@
</div>
</div><!-- .panel-body -->
</div>
{% endif %}
</div>
</div>
......
circle/dashboard/views/storage.py
......@@ -16,68 +16,150 @@
# with CIRCLE. If not, see <http://www.gnu.org/licenses/>.
from __future__ import unicode_literals, absolute_import
import errno
from django.contrib import messages
from django.core.urlresolvers import reverse
from django.db.models import Q
from django.utils.translation import ugettext_lazy as _
from django.views.generic import UpdateView
from django.views.generic import UpdateView, TemplateView
from braces.views import SuperuserRequiredMixin
from sizefield.utils import filesizeformat
from braces.views import SuperuserRequiredMixin
from common.models import WorkerNotFound
from storage.models import DataStore, Disk
from ..tables import DiskListTable
from ..forms import DataStoreForm, DiskForm
from ..forms import DataStoreForm, DiskForm
from django.shortcuts import get_object_or_404, redirect
from django.db import IntegrityError
class StorageDetail(SuperuserRequiredMixin, UpdateView):
model = DataStore
form_class = DataStoreForm
class StorageDetail(SuperuserRequiredMixin, TemplateView):
template_name = "dashboard/storage/detail.html"
def get_object(self):
return DataStore.objects.get()
def _current_querystring(self):
# Preserve UI state across redirects.
parts = []
ds = self.request.GET.get("ds")
if ds:
parts.append("ds=%s" % ds)
def get_context_data(self, **kwargs):
context = super(StorageDetail, self).get_context_data(**kwargs)
flt = self.request.GET.get("filter")
if flt:
parts.append("filter=%s" % flt)
s = self.request.GET.get("s")
if s:
parts.append("s=%s" % s)
return ("?" + "&".join(parts)) if parts else ""
def _redirect_with_ds(self, ds_pk):
# Redirect back to page selecting a specific datastore.
parts = ["ds=%s" % ds_pk]
flt = self.request.GET.get("filter")
if flt:
parts.append("filter=%s" % flt)
s = self.request.GET.get("s")
if s:
parts.append("s=%s" % s)
return redirect("%s?%s" % (self.request.path, "&".join(parts)))
def get_datastore(self):
ds_id = self.request.GET.get("ds")
qs = DataStore.objects.order_by("name")
# "new" (or empty) means create mode, no existing datastore.
if ds_id == "new":
return None
if ds_id:
return get_object_or_404(DataStore, pk=ds_id)
return qs.first() # can be None if empty
ds = self.get_object()
def post(self, request, *args, **kwargs):
ds_id = request.POST.get("ds") # hidden field in template: "new" or an id
instance = None
if ds_id and ds_id != "new":
instance = get_object_or_404(DataStore, pk=int(ds_id))
form = DataStoreForm(request.POST, instance=instance)
if not form.is_valid():
context = self.get_context_data()
context["form"] = form
return self.render_to_response(context)
try:
context['stats'] = self._get_stats()
context['missing_disks'] = ds.get_missing_disks()
context['orphan_disks'] = ds.get_orphan_disks()
except WorkerNotFound:
messages.error(self.request, _("The DataStore is offline."))
context['disk_table'] = DiskListTable(
self.get_table_data(), request=self.request,
template="django_tables2/with_pagination.html")
context['filter_names'] = (
('vm', _("virtual machine")),
('template', _("template")),
('none', _("none")),
)
return context
ds = form.save()
except IntegrityError as e:
messages.error(request, _("Could not save datastore: %s") % e)
return redirect(request.path)
messages.success(request, _("Datastore saved."))
return redirect("%s?ds=%s" % (request.path, ds.pk))
# def post(self, request, *args, **kwargs):
# action = request.POST.get("action")
# if action == "create":
# return self._handle_create(request)
# if action == "update":
# return self._handle_update(request)
# messages.error(request, _("Unknown action."))
# return redirect(request.path + self._current_querystring())
def _handle_create(self, request):
form = DataStoreCreateForm(request.POST)
if not form.is_valid():
context = self.get_context_data()
context["create_form"] = form
return self.render_to_response(context)
try:
ds = form.save()
except IntegrityError as e:
messages.error(request, _("Could not create datastore: %s") % e)
return redirect(request.path + self._current_querystring())
messages.success(request, _("Datastore created."))
return self._redirect_with_ds(ds.pk)
def _handle_update(self, request):
ds = self.get_datastore()
if ds is None:
messages.error(request, _("No datastore selected."))
return redirect(request.path)
form = DataStoreForm(request.POST, instance=ds)
if not form.is_valid():
context = self.get_context_data()
context["edit_form"] = form
return self.render_to_response(context)
try:
ds = form.save()
except IntegrityError as e:
messages.error(request, _("Could not update datastore: %s") % e)
return self._redirect_with_ds(ds.pk)
messages.success(request, _("Datastore updated."))
return self._redirect_with_ds(ds.pk)
def get_table_data(self, ds):
if ds is None:
return Disk.objects.none()
def get_table_data(self):
ds = self.get_object()
qs = Disk.objects.filter(datastore=ds, destroyed=None)
filter_name = self.request.GET.get("filter")
search = self.request.GET.get("s")
filter_queries = {
'vm': {
'instance_set__isnull': False,
},
'template': {
'template_set__isnull': False,
},
'none': {
'template_set__isnull': True,
'instance_set__isnull': True,
}
'vm': {'instance_set__isnull': False},
'template': {'template_set__isnull': False},
'none': {'template_set__isnull': True, 'instance_set__isnull': True},
}
if filter_name:
......@@ -90,21 +172,18 @@ class StorageDetail(SuperuserRequiredMixin, UpdateView):
return qs
def _get_stats(self):
# datastore stats
stats = self.object.get_statistics()
def get_stats(self, ds):
stats = ds.get_statistics()
free_space = int(stats['free_space'])
free_percent = float(stats['free_percent'])
total_space = free_space / (free_percent/100.0)
total_space = free_space / (free_percent / 100.0)
used_space = total_space - free_space
# file stats
data = self.get_object().get_file_statistics()
data = ds.get_file_statistics()
dumps_size = sum(d['size'] for d in data['dumps'])
trash = sum(d['size'] for d in data['trash'])
iso_raw = sum(d['size'] for d in data['disks']
if d['format'] in ("iso", "raw"))
iso_raw = sum(d['size'] for d in data['disks'] if d['format'] in ("iso", "raw"))
vm_size = vm_actual_size = template_actual_size = 0
for d in data['disks']:
......@@ -127,8 +206,45 @@ class StorageDetail(SuperuserRequiredMixin, UpdateView):
'template_actual_size': template_actual_size,
}
def get_success_url(self):
return reverse("dashboard.views.storage")
def get_context_data(self, **kwargs):
context = super(StorageDetail, self).get_context_data(**kwargs)
ds = self.get_datastore()
context["datastores"] = DataStore.objects.order_by("name")
context["ds"] = ds
# If no datastore exists yet, only show the create form.
if ds is not None:
context["form"] = DataStoreForm(instance=ds)
context["mode"] = "update"
context["ds_selected"] = str(ds.pk)
try:
context["stats"] = self.get_stats(ds)
context["missing_disks"] = ds.get_missing_disks()
context["orphan_disks"] = ds.get_orphan_disks()
except WorkerNotFound:
messages.error(self.request, _("The DataStore is offline."))
except (OSError, IOError) as e:
messages.error(self.request, e)
else:
context["form"] = DataStoreForm()
context["mode"] = "create"
context["ds_selected"] = "new"
context["stats"] = None
context["missing_disks"] = None
context["orphan_disks"] = None
context["disk_table"] = DiskListTable(
self.get_table_data(ds), request=self.request,
template="django_tables2/with_pagination.html"
)
context["filter_names"] = (
('vm', _("virtual machine")),
('template', _("template")),
('none', _("none")),
)
return context
class DiskDetail(SuperuserRequiredMixin, UpdateView):
......@@ -138,3 +254,4 @@ class DiskDetail(SuperuserRequiredMixin, UpdateView):
def form_valid(self, form):
pass
circle/fabfile.py
......@@ -37,7 +37,7 @@ except Exception as e:
else:
env.roledefs['node'] = [unicode(n.host.ipv4)
for n in _Node.objects.filter(enabled=True)]
env.roledefs['storage'] = [_DataStore.objects.get().hostname]
env.roledefs['storage'] = [_DataStore.objects.all()[0].hostname]
def update_all():
......
circle/storage/models.py
......@@ -47,14 +47,16 @@ class DataStore(Model):
"""Collection of virtual disks.
"""
name = CharField(max_length=100, unique=True, verbose_name=_('name'))
path = CharField(max_length=200, unique=True, verbose_name=_('path'))
hostname = CharField(max_length=40, unique=True,
verbose_name=_('hostname'))
path = CharField(max_length=200, verbose_name=_('path'))
hostname = CharField(max_length=40, verbose_name=_('hostname'))
class Meta:
ordering = ['name']
verbose_name = _('datastore')
verbose_name_plural = _('datastores')
unique_together = (
("hostname", "path"),
)
def __unicode__(self):
return u'%s (%s)' % (self.name, self.path)
......@@ -106,7 +108,7 @@ class DataStore(Model):
queue_name = self.get_remote_queue_name('storage', "slow")
files = set(storage_tasks.list_files.apply_async(
args=[self.path], queue=queue_name).get(timeout=timeout))
disks = Disk.objects.filter(destroyed__isnull=True, is_ready=True)
disks = Disk.objects.filter(datastore=self, destroyed__isnull=True, is_ready=True)
return disks.exclude(filename__in=files)
@method_cache(120)
......@@ -426,7 +428,7 @@ class Disk(TimeStampedModel):
@classmethod
def __create(cls, user, params):
datastore = params.pop('datastore', DataStore.objects.get())
datastore = params.pop('datastore', DataStore.objects.all()[0])
filename = params.pop('filename', str(uuid.uuid4()))
disk = cls(filename=filename, datastore=datastore, **params)
return disk
......
circle/storage/tasks/periodic_tasks.py
......@@ -65,6 +65,7 @@ def list_orphan_disks(timeout=15):
queue_name = ds.get_remote_queue_name('storage', "slow")
files = set(storage_tasks.list_files.apply_async(
args=[ds.path], queue=queue_name).get(timeout=timeout))
logging.error("files in %s: %s" % (ds.path, files))
disks = set([disk.filename for disk in ds.disk_set.all()])
for i in files - disks:
if not re.match('cloud-[0-9]*\.dump', i):
......
circle/vm/models/instance.py
......@@ -523,7 +523,7 @@ class Instance(AclBase, VirtualMachineDescModel, StatusModel, OperatedMixin,
datastore = self.disks.all()[0].datastore
except IndexError:
from storage.models import DataStore
datastore = DataStore.objects.get()
datastore = DataStore.objects.all()[0]
path = datastore.path + '/' + self.vm_name + '.dump'
return {'datastore': datastore, 'path': path}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment